00000800:00000200:3.1:1787759413.566170:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 142848 low water 294656 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.566188:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 142848 low water 294656 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.566202:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 163304 low water 284428 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.566214:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 251552 low water 256944 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.569230:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 306520 low water 229460 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.571602:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 50856 low water 357292 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.571625:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 50856 low water 357292 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.571632:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 50856 low water 357292 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.571665:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 222196 low water 288262 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.571670:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 222196 low water 288262 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.572601:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 222196 low water 288262 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.572628:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 220916 low water 288902 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.573238:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 383308 low water 224346 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.575020:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 90572 low water 370714 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.575049:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 90572 low water 370714 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.575060:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 90572 low water 370714 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.575066:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 90572 low water 370714 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.577159:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 224840 low water 320220 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.577171:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 224840 low water 320220 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.578428:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 451148 low water 223706 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.579387:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -60180 low water 479370 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.579406:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -61460 low water 480010 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.580932:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 218536 low water 356652 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.580953:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 218536 low water 356652 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.580990:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 218536 low water 356652 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.580997:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 218536 low water 356652 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.581984:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 218536 low water 356652 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.582008:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 217256 low water 357292 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.584407:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 540760 low water 228820 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.586526:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -34484 low water 516442 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.586547:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -34484 low water 516442 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.588317:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 103576 low water 447412 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.588338:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 103576 low water 447412 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.588344:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 103576 low water 447412 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.589387:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 264688 low water 383496 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.589405:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 264688 low water 383496 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.589415:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 263408 low water 384136 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.589728:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 263408 low water 384136 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.590992:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 613756 low water 242242 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.592412:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -25404 low water 561822 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.592430:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -25404 low water 561822 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.592952:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -26684 low water 562462 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.594435:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 299332 low water 416094 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.594455:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 299332 low water 416094 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.594466:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 299332 low water 416094 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.594472:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 299332 low water 416094 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.595889:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 299332 low water 416094 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.596436:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 397808 low water 383496 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.802872:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 80336 low water 592152 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.802889:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 80336 low water 592152 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.804886:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 276092 low water 510914 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.804906:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 276092 low water 510914 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.804911:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 276092 low water 510914 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.804943:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 437204 low water 446998 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.804948:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 437204 low water 446998 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.806379:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 628952 low water 351124 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.814036:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 56352 low water 670704 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.814057:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 56352 low water 670704 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.814069:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 56352 low water 670704 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.814074:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 56352 low water 670704 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.814121:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 280552 low water 575244 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.815010:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 280552 low water 575244 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.815025:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 280552 low water 575244 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.815035:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 280552 low water 575244 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.816093:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 279272 low water 575884 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.817147:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 440384 low water 511968 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.817181:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 440384 low water 511968 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.817187:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 440384 low water 511968 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.817236:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 729328 low water 384136 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.828794:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 288484 low water 671118 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.829095:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 288484 low water 671118 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.829106:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 288484 low water 671118 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.829115:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 288484 low water 671118 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.829121:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 288484 low water 671118 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.829450:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 288484 low water 671118 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.829462:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 288484 low water 671118 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.829470:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 288484 low water 671118 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.832039:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 352400 low water 639160 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.832052:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 352400 low water 639160 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.832092:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 577428 low water 543286 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.834370:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 163296 low water 766992 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.834388:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 163296 low water 766992 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.834398:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 163296 low water 766992 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.834403:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 163296 low water 766992 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.834425:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 388324 low water 671118 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.837090:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 388324 low water 671118 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.837111:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 388324 low water 671118 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.837123:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 388324 low water 671118 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.838671:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 388324 low water 671118 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759413.839298:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 710548 low water 543286 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.067880:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 498812 low water 798914 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.067901:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 498812 low water 798914 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.067912:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 498812 low water 798914 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.067932:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 498812 low water 798914 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.068899:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 690560 low water 703040 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.068914:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 690560 low water 703040 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.068924:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 690560 low water 703040 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.068929:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 690560 low water 703040 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.069242:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 690560 low water 703040 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.069256:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 690560 low water 703040 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.069265:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 690560 low water 703040 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.072845:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1365572 low water 415454 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.076052:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -40580 low water 1118530 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.076088:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -40580 low water 1118530 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.076103:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -40580 low water 1118530 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.076116:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -40580 low water 1118530 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.077835:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 23336 low water 1086572 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.077855:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 23336 low water 1086572 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.077866:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 23336 low water 1086572 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.077870:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 23336 low water 1086572 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.083666:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 120532 low water 1054614 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.083678:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 120532 low water 1054614 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.084966:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 248364 low water 990698 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.084987:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 248364 low water 990698 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.084993:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 248364 low water 990698 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.090025:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 634504 low water 830908 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.090064:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 634504 low water 830908 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.090070:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 634504 low water 830908 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.090439:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 634504 low water 830908 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.090454:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 634504 low water 830908 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.091487:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 634504 low water 830908 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.099113:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1309588 low water 543286 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.287969:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 121380 low water 1220590 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.288255:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 121380 low water 1220590 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.288285:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 121380 low water 1220590 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.288298:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 121380 low water 1220590 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.288310:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 121380 low water 1220590 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.290106:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 645496 low water 958532 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.290128:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 645496 low water 958532 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.290140:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 645496 low water 958532 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.290146:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 645496 low water 958532 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.290933:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 645496 low water 958532 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.290947:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 645496 low water 958532 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.290970:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 645496 low water 958532 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.292874:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 837244 low water 862658 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.292906:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 837244 low water 862658 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.292917:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 837244 low water 862658 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.292923:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 837244 low water 862658 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.293411:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 837244 low water 862658 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.293443:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 837244 low water 862658 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.293455:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 837244 low water 862658 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.294105:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 837244 low water 862658 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.294120:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 837244 low water 862658 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.295856:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1062272 low water 766784 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.542135:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 338792 low water 1278284 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.542159:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 338792 low water 1278284 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.542170:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 338792 low water 1278284 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.542176:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 338792 low water 1278284 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.542673:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 850120 low water 1022620 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.542688:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 850120 low water 1022620 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.542698:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 850120 low water 1022620 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.542703:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 850120 low water 1022620 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.543640:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 850120 low water 1022620 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.543659:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 850120 low water 1022620 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.543671:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 850120 low water 1022620 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.544130:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 850120 low water 1022620 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.544146:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 850120 low water 1022620 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.544157:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 850120 low water 1022620 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.544484:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 848840 low water 1023260 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.546553:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1298824 low water 831548 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.772157:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 477976 low water 1508212 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.772176:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 477976 low water 1508212 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.772458:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 477976 low water 1508212 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.772480:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 477976 low water 1508212 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.772750:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 477976 low water 1508212 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.772765:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 477976 low water 1508212 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.773031:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 477976 low water 1508212 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.773045:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 477976 low water 1508212 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.773449:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 477976 low water 1508212 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.773615:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 477976 low water 1508212 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.773794:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 477976 low water 1508212 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.774560:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 477976 low water 1508212 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.774731:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 476696 low water 1508852 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.779570:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 650596 low water 1438542 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.779594:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 650596 low water 1438542 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.779606:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 650596 low water 1438542 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.779611:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 650596 low water 1438542 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.779639:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 714512 low water 1406584 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.781849:0:32:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1453512 low water 1087004 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.987681:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 204896 low water 1827792 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.987696:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 204896 low water 1827792 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.987707:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 204896 low water 1827792 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.988013:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 204896 low water 1827792 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759414.989925:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1376104 low water 1342028 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759415.264631:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 741976 low water 1726164 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759415.264827:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 741976 low water 1726164 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759415.265745:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 741976 low water 1726164 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759415.265781:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 741976 low water 1726164 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759415.265795:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 741976 low water 1726164 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759415.265808:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 741976 low water 1726164 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759415.265819:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 741976 low water 1726164 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759415.265830:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 741976 low water 1726164 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759415.265840:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 741976 low water 1726164 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759415.266320:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 741976 low water 1726164 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759415.266332:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 741976 low water 1726164 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759415.266341:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 741976 low water 1726164 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759415.269162:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 805892 low water 1694206 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759415.269175:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 805892 low water 1694206 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759415.269246:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1701580 low water 1246362 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.490606:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -36524 low water 2115414 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.490790:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -36524 low water 2115414 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.490977:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -36524 low water 2115414 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.491157:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -36524 low water 2115414 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.491339:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -36524 low water 2115414 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.495088:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 679340 low water 1757482 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.495969:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 679340 low water 1757482 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.496169:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 679340 low water 1757482 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.496730:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 679340 low water 1757482 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.497073:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 679340 low water 1757482 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.498109:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 679340 low water 1757482 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.499154:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 679340 low water 1757482 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.499189:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 679340 low water 1757482 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.499201:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 679340 low water 1757482 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.499209:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 679340 low water 1757482 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.499221:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 679340 low water 1757482 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.499262:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 743256 low water 1725524 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.499276:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 807172 low water 1693566 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.499291:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 935004 low water 1629650 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.500551:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1062420 low water 1565942 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.500566:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1062420 low water 1565942 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.500574:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1062420 low water 1565942 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.500579:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1062420 low water 1565942 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.500692:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1126336 low water 1533984 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.502625:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1190252 low water 1502026 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.502644:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1190252 low water 1502026 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.502656:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1190252 low water 1502026 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.502662:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1190252 low water 1502026 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.504167:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1318084 low water 1438110 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.504187:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1318084 low water 1438110 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.504196:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1318084 low water 1438110 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.504201:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1318084 low water 1438110 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759417.504238:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1509832 low water 1342236 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759419.280330:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 663892 low water 1765206 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759419.280352:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 663892 low water 1765206 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759419.280363:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 663892 low water 1765206 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759419.280370:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 663892 low water 1765206 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759419.280389:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 727808 low water 1733248 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759419.280430:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1190252 low water 1502026 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759419.281075:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1190252 low water 1502026 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759419.281092:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1190252 low water 1502026 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759419.281103:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1190252 low water 1502026 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759419.281504:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1190252 low water 1502026 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759419.281517:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1190252 low water 1502026 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759419.281528:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1190252 low water 1502026 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759419.283065:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1190252 low water 1502026 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759419.283084:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1190252 low water 1502026 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759419.283093:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1190252 low water 1502026 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759419.284574:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1254168 low water 1470068 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759419.284594:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1254168 low water 1470068 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759419.284604:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1254168 low water 1470068 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759419.284610:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1254168 low water 1470068 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759419.284645:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1765496 low water 1214404 conn ffff9751eb95c000 blocked idle queued
00010000:00000010:3.1:1787759442.867980:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183300.
00010000:00000010:3.1:1787759442.868020:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441e00.
00010000:00000010:3.1:1787759442.887224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f129e000.
00010000:00000010:3.1:1787759450.866510:0:274302:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183200.
00010000:00000010:3.1:1787759450.866522:0:274302:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440400.
00000080:00000010:3.1:1787759455.108855:0:284441:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3620.
00010000:00000010:3.1:1787759482.007414:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183000.
00010000:00000010:3.1:1787759482.007439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441400.
00000080:00000010:3.1:1787759482.076253:0:285367:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3120.
00000080:00000010:3.1:1787759482.633977:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5ea0.
00010000:00000010:3.1:1787759507.244174:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f129e500.
00010000:00000010:3.1:1787759507.244193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ec00.
00010000:00000010:3.1:1787759517.618388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104da00.
00010000:00000010:3.1:1787759517.618402:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44b7800.
00010000:00000010:3.1:1787759517.618415:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441600.
00010000:00000010:3.1:1787759517.618420:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30b00.
00010000:00000010:3.1:1787759517.618423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f2c00.
00000080:00000010:3.1:1787759517.645465:0:286146:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca660.
00000080:00000010:3.1:1787759517.645491:0:286146:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18ce300.
00000080:00000010:3.1:1787759565.711387:0:288327:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d400.
00010000:00000010:3.1:1787759565.711396:0:288327:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1e29300.
00010000:00000010:3.1:1787759565.711399:0:288327:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f3e00.
00010000:00000010:3.1:1787759565.731212:0:288327:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27800.
00000080:00000010:3.1:1787759566.442194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4820.
00010000:00000010:3.1:1787759588.251261:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27d00.
00010000:00000010:3.1:1787759588.251280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104da00.
00010000:00000010:3.1:1787759588.275197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27600.
00000800:00000200:3.1:1787759593.455183:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -40364 low water 2117334 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.456760:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -2012 low water 2098158 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.456779:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -2012 low water 2098158 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.456794:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -3292 low water 2098798 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.456800:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace -3292 low water 2098798 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.458008:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 23552 low water 2085376 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.458039:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 23552 low water 2085376 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.458046:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 23552 low water 2085376 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.462774:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 166732 low water 2013786 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.462795:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 166732 low water 2013786 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.463111:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 165452 low water 2014426 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.463125:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 165452 low water 2014426 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.463139:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 164172 low water 2015066 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.463146:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 164172 low water 2015066 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.463622:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 164172 low water 2015066 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.463639:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 162892 low water 2015706 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.463942:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 162892 low water 2015706 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.463958:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 161612 low water 2016346 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.465032:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 338024 low water 1928140 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.465050:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 338024 low water 1928140 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.465060:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 338024 low water 1928140 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.465065:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 338024 low water 1928140 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.466236:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 459468 low water 1867418 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.466260:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 459468 low water 1867418 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.466265:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 459468 low water 1867418 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.466977:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 615424 low water 1789440 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.466993:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 615424 low water 1789440 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.466998:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 615424 low water 1789440 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.467711:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 615424 low water 1789440 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.467751:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 615424 low water 1789440 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.468230:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 615424 low water 1789440 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.468250:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 615424 low water 1789440 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.468261:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 615424 low water 1789440 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.470900:0:32:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 614144 low water 1790080 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.471989:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 997640 low water 1598332 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.472005:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 997640 low water 1598332 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.472015:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 997640 low water 1598332 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.472020:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 997640 low water 1598332 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.480028:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1199200 low water 1497552 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.480181:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1254168 low water 1470068 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.480195:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1254168 low water 1470068 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.480205:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1254168 low water 1470068 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.480210:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1254168 low water 1470068 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.480230:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1382000 low water 1406152 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.481196:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1382000 low water 1406152 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.481213:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1382000 low water 1406152 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.481223:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1382000 low water 1406152 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.481755:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1382000 low water 1406152 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.481768:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1382000 low water 1406152 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.481790:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1382000 low water 1406152 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.482169:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1382000 low water 1406152 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.482183:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1382000 low water 1406152 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.482207:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1382000 low water 1406152 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.482518:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1382000 low water 1406152 conn ffff9751eb95c000 blocked idle queued
00000800:00000200:3.1:1787759593.486179:0:0:0:(socklnd_lib.c:496:ksocknal_write_space()) sk ffff9751da501500 wspace 1509832 low water 1342236 conn ffff9751eb95c000 blocked idle queued
00000080:00000010:3.1:1787759594.032654:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5a80.
00010000:00000010:3.1:1787759599.691277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6afe00.
00010000:00000010:3.1:1787759599.691288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104da00.
00010000:00000010:3.1:1787759599.691292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48600.
00010000:00000010:3.1:1787759599.691296:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4b600.
00010000:00000010:3.1:1787759599.691300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440200.
00010000:00000010:3.1:1787759599.691304:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4e00.
00010000:00000010:3.1:1787759599.691306:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e200.
00000080:00000010:3.1:1787759599.714184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4500.
00000080:00000010:3.1:1787759599.714245:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e4200.
00010000:00000010:3.1:1787759628.063561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4bb00.
00010000:00000010:3.1:1787759628.063572:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440e00.
00010000:00000010:3.1:1787759628.073629:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27f00.
00010000:00000010:3.1:1787759658.635612:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f000.
00010000:00000010:3.1:1787759697.900448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49600.
00010000:00000010:3.1:1787759698.304824:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f000.
00010000:00000010:3.1:1787759701.298242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846da00.
00010000:00000010:3.1:1787759703.262993:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00010000:00000010:3.1:1787759704.281439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c400.
00010000:00000010:3.1:1787759706.671245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440400.
00010000:00000010:3.1:1787759713.280334:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ca00.
00010000:00000010:3.1:1787759715.443826:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e800.
00010000:00000010:3.1:1787759724.042430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846cc00.
00010000:00000010:3.1:1787759726.215466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440800.
00010000:00000010:3.1:1787759727.855324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441a00.
00010000:00000010:3.1:1787759740.767364:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440c00.
00010000:00000010:3.1:1787759744.590508:0:1365:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441400.
00010000:00000010:3.1:1787759747.501109:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539fa00.
00010000:00000010:3.1:1787759754.268385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c000.
00010000:00000010:3.1:1787759755.340504:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846de00.
00010000:00000010:3.1:1787759769.086430:0:293271:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d000.
00010000:00000010:3.1:1787759781.289085:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49a00.
00010000:00000010:3.1:1787759782.533188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846da00.
00010000:00000010:3.1:1787759786.693618:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48400.
00010000:00000010:3.1:1787759788.907046:0:293271:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49a00.
00010000:00000010:3.1:1787759789.712510:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440c00.
00010000:00000010:3.1:1787759792.746180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846cc00.
00010000:00000010:3.1:1787759798.527175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d200.
00010000:00000010:3.1:1787759798.705683:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e200.
00010000:00000010:3.1:1787759799.236194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846dc00.
00010000:00000010:3.1:1787759799.379364:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c000.
00010000:00000010:3.1:1787759800.884171:0:293271:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441600.
00010000:00000010:3.1:1787759811.326372:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ec00.
00010000:00000010:3.1:1787759811.535901:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00010000:00000010:3.1:1787759825.664973:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c400.
00010000:00000010:3.1:1787759826.968243:0:293271:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440200.
00010000:00000010:3.1:1787759831.304834:0:293271:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00010000:00000010:3.1:1787759832.841305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49c00.
00010000:00000010:3.1:1787759833.699364:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846dc00.
00010000:00000010:3.1:1787759837.401077:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846de00.
00010000:00000010:3.1:1787759838.590179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ec00.
00010000:00000010:3.1:1787759843.476290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ea00.
00010000:00000010:3.1:1787759845.887193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ca00.
00010000:00000010:3.1:1787759851.392370:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d000.
00010000:00000010:3.1:1787759852.828704:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48600.
00010000:00000010:3.1:1787759855.351417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846dc00.
00010000:00000010:3.1:1787759856.797180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846cc00.
00010000:00000010:3.1:1787759859.403603:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c400.
00010000:00000010:3.1:1787759862.693342:0:293271:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440200.
00010000:00000010:3.1:1787759864.755205:0:293271:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846de00.
00010000:00000010:3.1:1787759867.205294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48600.
00010000:00000010:3.1:1787759872.046697:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48600.
00010000:00000010:3.1:1787759873.715192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441a00.
00010000:00000010:3.1:1787759876.179192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441c00.
00010000:00000010:3.1:1787759878.395285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c000.
00010000:00000010:3.1:1787759880.490420:0:257571:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49600.
00010000:00000010:3.1:1787759881.574185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00010000:00000010:3.1:1787759882.671192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49c00.
00010000:00000010:3.1:1787759883.093919:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441600.
00010000:00000010:3.1:1787759883.251181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49c00.
00010000:00000010:3.1:1787759884.803465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d200.
00010000:00000010:3.1:1787759886.932853:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d200.
00010000:00000010:3.1:1787759887.476456:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ca00.
00010000:00000010:3.1:1787759890.315051:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441400.
00010000:00000010:3.1:1787759890.778473:0:293271:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e800.
00010000:00000010:3.1:1787759894.856687:0:293271:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440200.
00010000:00000010:3.1:1787759896.047027:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441400.
00010000:00000010:3.1:1787759896.177376:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f800.
00010000:00000010:3.1:1787759897.017179:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48600.
00010000:00000010:3.1:1787759897.975956:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441800.
00010000:00000010:3.1:1787759902.592194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c000.
00010000:00000010:3.1:1787759903.231207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441a00.
00010000:00000010:3.1:1787759904.714982:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e200.
00010000:00000010:3.1:1787759908.299591:0:293271:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440600.
00010000:00000010:3.1:1787759910.882178:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ee00.
00010000:00000010:3.1:1787759912.505706:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d600.
00010000:00000010:3.1:1787759912.885639:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c400.
00010000:00000010:3.1:1787759913.138176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d800.
00010000:00000010:3.1:1787759915.992337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440400.
00000080:00000010:3.1:1787759918.714189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4780.
00010000:00000010:3.1:1787759919.930645:0:257578:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ec00.
00010000:00000010:3.1:1787759922.331177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846da00.
00010000:00000010:3.1:1787759926.535178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e800.
00010000:00000010:3.1:1787759933.255696:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49a00.
00010000:00000010:3.1:1787759935.441222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c000.
00010000:00000010:3.1:1787759941.998060:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ca00.
00010000:00000010:3.1:1787759954.893756:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ca00.
00010000:00000010:3.1:1787759957.430697:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ec00.
00010000:00000010:3.1:1787759958.635463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441c00.
00010000:00000010:3.1:1787759959.189883:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846de00.
00010000:00000010:3.1:1787759959.289189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f400.
00010000:00000010:3.1:1787759960.319366:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ea00.
00010000:00000010:3.1:1787759961.657784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f800.
00010000:00000010:3.1:1787759963.466391:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d200.
00010000:00000010:3.1:1787759966.054302:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49c00.
00010000:00000010:3.1:1787759967.595353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846cc00.
00010000:00000010:3.1:1787759968.783824:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48600.
00010000:00000010:3.1:1787759969.174207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49c00.
00010000:00000010:3.1:1787759970.990968:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440400.
00010000:00000010:3.1:1787759971.613192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441600.
00010000:00000010:3.1:1787759974.428646:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440a00.
00010000:00000010:3.1:1787759976.098302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539fc00.
00010000:00000010:3.1:1787759979.048593:0:257578:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846de00.
00010000:00000010:3.1:1787759980.848263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00010000:00000010:3.1:1787759982.758339:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c400.
00010000:00000010:3.1:1787759986.373445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48800.
00010000:00000010:3.1:1787759988.548346:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f000.
00010000:00000010:3.1:1787759997.862445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d800.
00010000:00000010:3.1:1787760004.812217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c200.
00010000:00000010:3.1:1787760005.616206:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440400.
00010000:00000010:3.1:1787760014.238275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e800.
00010000:00000010:3.1:1787760014.468366:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d400.
00010000:00000010:3.1:1787760018.721186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49a00.
00000080:00000010:3.1:1787760021.139426:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a840.
00010000:00000010:3.1:1787760023.690310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d000.
00010000:00000010:3.1:1787760025.535471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539fc00.
00010000:00000010:3.1:1787760030.444399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846da00.
00010000:00000010:3.1:1787760030.596050:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d400.
00010000:00000010:3.1:1787760037.701868:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ea00.
00010000:00000010:3.1:1787760039.337600:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d800.
00010000:00000010:3.1:1787760042.659172:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440600.
00010000:00000010:3.1:1787760046.009216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440000.
00010000:00000010:3.1:1787760046.188240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d400.
00010000:00000010:3.1:1787760047.292196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c600.
00010000:00000010:3.1:1787760048.037175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c000.
00010000:00000010:3.1:1787760048.863177:0:293271:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539fc00.
00010000:00000010:3.1:1787760050.982604:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d000.
00010000:00000010:3.1:1787760053.188188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846dc00.
00010000:00000010:3.1:1787760055.259338:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d400.
00010000:00000010:3.1:1787760055.747306:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49000.
00010000:00000010:3.1:1787760056.286183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441e00.
00010000:00000010:3.1:1787760056.806334:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d200.
00010000:00000010:3.1:1787760060.059011:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d600.
00010000:00000010:3.1:1787760060.111670:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48400.
00010000:00000010:3.1:1787760060.293946:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48400.
00010000:00000010:3.1:1787760066.607539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441800.
00010000:00000010:3.1:1787760068.442573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440c00.
00010000:00000010:3.1:1787760068.577342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f200.
00010000:00000010:3.1:1787760068.950605:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f000.
00010000:00000010:3.1:1787760070.233647:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e400.
00010000:00000010:3.1:1787760074.895183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440600.
00010000:00000010:3.1:1787760082.943011:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440a00.
00010000:00000010:3.1:1787760083.505936:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846da00.
00010000:00000010:3.1:1787760085.989843:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440800.
00010000:00000010:3.1:1787760088.045181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49600.
00010000:00000010:3.1:1787760088.249722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441400.
00010000:00000010:3.1:1787760088.958170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ec00.
00010000:00000010:3.1:1787760089.035175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c000.
00010000:00000010:3.1:1787760090.428504:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441e00.
00010000:00000010:3.1:1787760094.631977:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c000.
00010000:00000010:3.1:1787760094.776197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d600.
00010000:00000010:3.1:1787760096.634120:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440200.
00010000:00000010:3.1:1787760097.184294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441400.
00010000:00000010:3.1:1787760098.826476:0:257579:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846de00.
00010000:00000010:3.1:1787760105.349226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441a00.
00010000:00000010:3.1:1787760105.705733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49000.
00010000:00000010:3.1:1787760106.586268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440a00.
00010000:00000010:3.1:1787760107.333188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e400.
00010000:00000010:3.1:1787760108.047154:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e200.
00010000:00000010:3.1:1787760110.506177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846de00.
00010000:00000010:3.1:1787760112.092073:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48000.
00010000:00000010:3.1:1787760116.696869:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440a00.
00010000:00000010:3.1:1787760118.254317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f400.
00010000:00000010:3.1:1787760119.174327:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48400.
00010000:00000010:3.1:1787760121.009342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c200.
00010000:00000010:3.1:1787760126.136660:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ec00.
00010000:00000010:3.1:1787760127.481270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c200.
00010000:00000010:3.1:1787760130.717698:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440200.
00010000:00000010:3.1:1787760134.952464:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c200.
00010000:00000010:3.1:1787760136.140216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539fc00.
00010000:00000010:3.1:1787760139.391701:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440800.
00010000:00000010:3.1:1787760141.992303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d000.
00010000:00000010:3.1:1787760145.005450:0:293271:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c400.
00010000:00000010:3.1:1787760149.975177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c000.
00010000:00000010:3.1:1787760155.910567:0:257571:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846de00.
00010000:00000010:3.1:1787760156.743553:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48a00.
00010000:00000010:3.1:1787760157.265595:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440400.
00010000:00000010:3.1:1787760160.076515:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49800.
00010000:00000010:3.1:1787760160.429171:0:257571:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c000.
00010000:00000010:3.1:1787760164.277765:0:257571:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440a00.
00010000:00000010:3.1:1787760165.120572:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e800.
00010000:00000010:3.1:1787760165.397159:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846dc00.
00010000:00000010:3.1:1787760166.158770:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441a00.
00010000:00000010:3.1:1787760177.280712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441e00.
00010000:00000010:3.1:1787760184.173232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846da00.
00010000:00000010:3.1:1787760185.511012:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539fc00.
00010000:00000010:3.1:1787760188.847483:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440200.
00010000:00000010:3.1:1787760188.896315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c400.
00010000:00000010:3.1:1787760192.083187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c200.
00010000:00000010:3.1:1787760193.780343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ec00.
00010000:00000010:3.1:1787760198.974465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d200.
00010000:00000010:3.1:1787760201.167719:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48a00.
00010000:00000010:3.1:1787760206.056733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49000.
00010000:00000010:3.1:1787760209.403189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846dc00.
00010000:00000010:3.1:1787760212.653297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846da00.
00010000:00000010:3.1:1787760218.105185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49c00.
00010000:00000010:3.1:1787760218.535754:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49c00.
00010000:00000010:3.1:1787760218.916183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e600.
00010000:00000010:3.1:1787760221.113702:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441200.
00010000:00000010:3.1:1787760222.727554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846de00.
00010000:00000010:3.1:1787760223.572233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e600.
00010000:00000010:3.1:1787760227.070110:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48600.
00010000:00000010:3.1:1787760237.154242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c800.
00010000:00000010:3.1:1787760237.604826:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441e00.
00010000:00000010:3.1:1787760238.475189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c400.
00010000:00000010:3.1:1787760240.445275:0:293271:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846de00.
00010000:00000010:3.1:1787760240.576503:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49000.
00010000:00000010:3.1:1787760243.240655:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846cc00.
00010000:00000010:3.1:1787760243.340682:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440a00.
00010000:00000010:3.1:1787760243.969458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f800.
00010000:00000010:3.1:1787760245.202961:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c600.
00010000:00000010:3.1:1787760245.725333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c600.
00010000:00000010:3.1:1787760250.121219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e600.
00010000:00000010:3.1:1787760250.714942:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00010000:00000010:3.1:1787760252.332572:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441000.
00010000:00000010:3.1:1787760258.668063:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c600.
00010000:00000010:3.1:1787760258.800285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c400.
00010000:00000010:3.1:1787760259.789751:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e200.
00010000:00000010:3.1:1787760261.017382:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441e00.
00010000:00000010:3.1:1787760264.612398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49000.
00010000:00000010:3.1:1787760269.726977:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c400.
00010000:00000010:3.1:1787760276.031472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e600.
00010000:00000010:3.1:1787760280.902188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48600.
00010000:00000010:3.1:1787760283.533262:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441000.
00010000:00000010:3.1:1787760288.509367:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e400.
00010000:00000010:3.1:1787760288.778814:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e200.
00010000:00000010:3.1:1787760300.108292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d400.
00010000:00000010:3.1:1787760300.239764:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846de00.
00010000:00000010:3.1:1787760301.700911:0:257578:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f800.
00010000:00000010:3.1:1787760302.531432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c800.
00010000:00000010:3.1:1787760307.128186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440000.
00010000:00000010:3.1:1787760307.148406:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49000.
00010000:00000010:3.1:1787760307.926767:0:293271:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846dc00.
00010000:00000010:3.1:1787760308.158284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48600.
00010000:00000010:3.1:1787760308.604040:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d200.
00010000:00000010:3.1:1787760311.252937:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48000.
00010000:00000010:3.1:1787760311.425368:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440800.
00010000:00000010:3.1:1787760313.373651:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d800.
00010000:00000010:3.1:1787760317.315841:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441400.
00010000:00000010:3.1:1787760321.855929:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c600.
00010000:00000010:3.1:1787760325.135438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f800.
00010000:00000010:3.1:1787760325.488621:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c400.
00010000:00000010:3.1:1787760328.544194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49600.
00010000:00000010:3.1:1787760329.364456:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846cc00.
00010000:00000010:3.1:1787760331.054318:0:293271:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e400.
00010000:00000010:3.1:1787760331.437584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e200.
00010000:00000010:3.1:1787760332.492290:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c400.
00010000:00000010:3.1:1787760332.560182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441600.
00010000:00000010:3.1:1787760338.081714:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49800.
00010000:00000010:3.1:1787760339.082196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48000.
00010000:00000010:3.1:1787760339.236242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48000.
00010000:00000010:3.1:1787760350.132711:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ca00.
00010000:00000010:3.1:1787760350.870187:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441000.
00010000:00000010:3.1:1787760353.144331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48000.
00010000:00000010:3.1:1787760356.461649:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f000.
00010000:00000010:3.1:1787760361.061616:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e200.
00010000:00000010:3.1:1787760368.552403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440200.
00010000:00000010:3.1:1787760369.428404:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441800.
00010000:00000010:3.1:1787760369.641473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ea00.
00010000:00000010:3.1:1787760372.690465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846cc00.
00010000:00000010:3.1:1787760374.513309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d800.
00010000:00000010:3.1:1787760376.375295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f800.
00010000:00000010:3.1:1787760379.571181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d200.
00010000:00000010:3.1:1787760379.615668:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846de00.
00010000:00000010:3.1:1787760380.613452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f000.
00010000:00000010:3.1:1787760381.346564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e200.
00010000:00000010:3.1:1787760382.761954:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d800.
00010000:00000010:3.1:1787760402.033471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c200.
00010000:00000010:3.1:1787760403.455066:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846da00.
00010000:00000010:3.1:1787760408.858305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440800.
00010000:00000010:3.1:1787760413.609469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ee00.
00010000:00000010:3.1:1787760414.283323:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e400.
00010000:00000010:3.1:1787760425.661274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440c00.
00010000:00000010:3.1:1787760427.630585:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d000.
00010000:00000010:3.1:1787760428.962180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00010000:00000010:3.1:1787760430.420222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f400.
00010000:00000010:3.1:1787760443.458304:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f200.
00010000:00000010:3.1:1787760443.543207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48400.
00010000:00000010:3.1:1787760447.266466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440c00.
00010000:00000010:3.1:1787760451.139210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539fc00.
00010000:00000010:3.1:1787760458.253227:0:257571:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441400.
00010000:00000010:3.1:1787760462.445182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ea00.
00010000:00000010:3.1:1787760462.540303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49200.
00010000:00000010:3.1:1787760471.190501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49800.
00010000:00000010:3.1:1787760472.423437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48600.
00010000:00000010:3.1:1787760473.092926:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c600.
00010000:00000010:3.1:1787760475.996405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846de00.
00010000:00000010:3.1:1787760479.549046:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f200.
00010000:00000010:3.1:1787760522.744158:0:294596:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49200.
00010000:00000010:3.1:1787760523.053238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f800.
00010000:00000010:3.1:1787760524.642679:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49200.
00010000:00000010:3.1:1787760526.201315:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742100.
00010000:00000010:3.1:1787760527.505482:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742600.
00010000:00000010:3.1:1787760527.505497:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d000.
00010000:00000010:3.1:1787760529.053197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0700.
00010000:00000010:3.1:1787760540.300331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c200.
00010000:00000010:3.1:1787760562.882575:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0100.
00010000:00000010:3.1:1787760562.882587:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49600.
00010000:00000010:3.1:1787760580.107596:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846cc00.
00010000:00000010:3.1:1787760588.265480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f59800.
00000080:00000010:3.1:1787760588.265487:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca1a0.
00010000:00000010:3.1:1787760613.881194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742800.
00010000:00000010:3.1:1787760614.183811:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440a00.
00010000:00000010:3.1:1787760614.183821:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441400.
00000080:00000010:3.1:1787760614.292959:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cac20.
00000080:00000010:3.1:1787760614.390843:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303b20.
00000080:00000010:3.1:1787760614.390858:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca900.
00010000:00000010:3.1:1787760614.492542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441c00.
00010000:00000010:3.1:1787760615.000456:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518d400.
00010000:00000010:3.1:1787760615.000477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f600.
00000080:00000010:3.1:1787760615.063482:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303bc0.
00010000:00000010:3.1:1787760615.424351:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58e00.
00000080:00000010:3.1:1787760615.713262:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9200.
00010000:00000010:3.1:1787760615.805260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843fe00.
00010000:00000010:3.1:1787760616.066096:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843ea00.
00010000:00000010:3.1:1787760616.066126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843f800.
00000080:00000010:3.1:1787760616.727433:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9de0.
00000080:00000010:3.1:1787760616.727443:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e95e0.
00010000:00000010:3.1:1787760617.191108:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f800.
00010000:00000010:3.1:1787760617.214356:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58e00.
00000080:00000010:3.1:1787760617.782181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303e40.
00000080:00000010:3.1:1787760617.952420:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9980.
00000080:00000010:3.1:1787760618.331273:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5160.
00000080:00000010:3.1:1787760618.331298:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18b98c0.
00010000:00000010:3.1:1787760628.463247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58e00.
00000080:00000010:3.1:1787760628.463265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca3c0.
00010000:00000010:3.1:1787760628.473559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58c00.
00000080:00000010:3.1:1787760628.473571:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca880.
00010000:00000010:3.1:1787760628.782240:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90000.
00010000:00000010:3.1:1787760628.905795:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90500.
00010000:00000010:3.1:1787760629.071251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58200.
00000080:00000010:3.1:1787760629.071258:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca8c0.
00000080:00000010:3.1:1787760629.186976:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303ee0.
00010000:00000010:3.1:1787760629.420176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58200.
00000080:00000010:3.1:1787760629.420184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca8c0.
00010000:00000010:3.1:1787760629.714799:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843ee00.
00010000:00000010:3.1:1787760629.714806:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6db00.
00010000:00000010:3.1:1787760629.714811:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440c00.
00010000:00000010:3.1:1787760629.915580:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440200.
00010000:00000010:3.1:1787760629.915587:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440600.
00010000:00000010:3.1:1787760629.920501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f59c00.
00000080:00000010:3.1:1787760629.920515:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5040.
00000080:00000010:3.1:1787760629.920534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9020.
00010000:00000010:3.1:1787760629.940830:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58200.
00000080:00000010:3.1:1787760630.579354:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9a20.
00010000:00000010:3.1:1787760630.765230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f400.
00010000:00000010:3.1:1787760634.335962:0:1381:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90900.
00010000:00000010:3.1:1787760634.335973:0:1381:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843fc00.
00010000:00000010:3.1:1787760641.088185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d200.
00010000:00000010:3.1:1787760641.088214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4e17600.
00010000:00000010:3.1:1787760641.088219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440800.
00010000:00000010:3.1:1787760660.746702:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58200.
00010000:00000010:3.1:1787760670.229312:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518df00.
00010000:00000010:3.1:1787760670.229331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e600.
00000080:00000010:3.1:1787760670.595197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303880.
00000080:00000010:3.1:1787760670.595230:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d8cbf380.
00010000:00000010:3.1:1787760670.595236:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3908800.
00010000:00000010:3.1:1787760670.595251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440e00.
00000080:00000010:3.1:1787760670.609521:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc260.
00000080:00000010:3.1:1787760670.609549:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1ae18c0.
00010000:00000010:3.1:1787760670.609562:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1100.
00010000:00000010:3.1:1787760670.609569:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c600.
00000080:00000010:3.1:1787760670.639499:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303fa0.
00000080:00000010:3.1:1787760671.006443:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc6e0.
00000080:00000010:3.1:1787760671.006474:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18ba940.
00010000:00000010:3.1:1787760671.006483:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47200.
00010000:00000010:3.1:1787760671.006488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed9e00.
00000080:00000010:3.1:1787760671.104677:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303b40.
00000080:00000010:3.1:1787760671.110476:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18bca40.
00010000:00000010:3.1:1787760671.110497:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1300.
00010000:00000010:3.1:1787760671.110503:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c800.
00000080:00000010:3.1:1787760671.141953:0:257570:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303a00.
00000080:00000010:3.1:1787760671.141984:0:257570:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d8cbc200.
00010000:00000010:3.1:1787760671.141992:0:257570:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183700.
00010000:00000010:3.1:1787760671.141996:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814200.
00000080:00000010:3.1:1787760671.395224:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5c40.
00000080:00000010:3.1:1787760671.395263:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c21ed280.
00010000:00000010:3.1:1787760671.395274:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4400.
00010000:00000010:3.1:1787760671.395278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814800.
00000080:00000010:3.1:1787760671.425173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303da0.
00000080:00000010:3.1:1787760671.425201:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d115b180.
00010000:00000010:3.1:1787760671.425247:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518d800.
00010000:00000010:3.1:1787760671.425255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e200.
00000080:00000010:3.1:1787760671.453669:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303760.
00000080:00000010:3.1:1787760671.453694:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18cb9c0.
00010000:00000010:3.1:1787760671.453703:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4bd00.
00010000:00000010:3.1:1787760671.453707:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe469400.
00000080:00000010:3.1:1787760671.475348:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcee0.
00000080:00000010:3.1:1787760671.475373:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1be9080.
00010000:00000010:3.1:1787760671.478298:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518d400.
00010000:00000010:3.1:1787760671.478324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104da00.
00000080:00000010:3.1:1787760671.489368:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303b60.
00000080:00000010:3.1:1787760671.489387:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18c8840.
00000080:00000010:3.1:1787760671.630409:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5dc0.
00010000:00000010:3.1:1787760671.738271:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4e17d00.
00010000:00000010:3.1:1787760671.738279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56c00.
00000080:00000010:3.1:1787760671.750912:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcd00.
00000080:00000010:3.1:1787760671.751843:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e39c0.
00010000:00000010:3.1:1787760671.760697:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4300.
00010000:00000010:3.1:1787760671.760715:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814400.
00000080:00000010:3.1:1787760671.773343:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303780.
00000080:00000010:3.1:1787760671.776299:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d115eb40.
00010000:00000010:3.1:1787760675.793876:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35da00.
00010000:00000010:3.1:1787760675.793891:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783aa00.
00010000:00000010:3.1:1787760676.027913:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742700.
00010000:00000010:3.1:1787760676.027920:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783a600.
00010000:00000010:3.1:1787760676.027924:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90200.
00010000:00000010:3.1:1787760676.027927:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783b600.
00010000:00000010:3.1:1787760676.040187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742900.
00010000:00000010:3.1:1787760676.040196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783b000.
00010000:00000010:3.1:1787760676.052312:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fa00.
00010000:00000010:3.1:1787760676.052333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798a800.
00010000:00000010:3.1:1787760676.086044:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90500.
00010000:00000010:3.1:1787760676.086082:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783a000.
00010000:00000010:3.1:1787760676.456549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101a00.
00010000:00000010:3.1:1787760676.456563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b600.
00010000:00000010:3.1:1787760676.456569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101300.
00010000:00000010:3.1:1787760676.456573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b200.
00010000:00000010:3.1:1787760676.456575:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101500.
00010000:00000010:3.1:1787760676.456579:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798a600.
00010000:00000010:3.1:1787760676.456584:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1e29000.
00010000:00000010:3.1:1787760676.456586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b800.
00010000:00000010:3.1:1787760676.478928:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1e29800.
00010000:00000010:3.1:1787760676.478937:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6de00.
00010000:00000010:3.1:1787760676.516435:0:302236:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1000.
00010000:00000010:3.1:1787760676.516443:0:302236:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2400.
00010000:00000010:3.1:1787760681.400289:0:257571:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9600.
00010000:00000010:3.1:1787760681.400303:0:257571:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3400.
00010000:00000010:3.1:1787760681.400306:0:257571:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9400.
00010000:00000010:3.1:1787760681.400310:0:257571:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888e00.
00010000:00000010:3.1:1787760681.400313:0:257571:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9d00.
00010000:00000010:3.1:1787760681.400316:0:257571:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7889400.
00010000:00000010:3.1:1787760681.400320:0:257571:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19e00.
00010000:00000010:3.1:1787760681.400323:0:257571:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888600.
00010000:00000010:3.1:1787760681.423478:0:302594:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9700.
00010000:00000010:3.1:1787760681.423485:0:302594:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7889000.
00010000:00000010:3.1:1787760683.432184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6700.
00010000:00000010:3.1:1787760683.432198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0300.
00010000:00000010:3.1:1787760683.432203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6c600.
00000080:00000010:3.1:1787760683.504311:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9260.
00010000:00000010:3.1:1787760683.821283:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d200.
00010000:00000010:3.1:1787760683.821300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa38e00.
00000080:00000010:3.1:1787760684.286226:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303b60.
00000080:00000010:3.1:1787760691.011239:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9a20.
00000080:00000010:3.1:1787760691.015753:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c21ec200.
00000080:00000010:3.1:1787760691.555407:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303440.
00000080:00000010:3.1:1787760691.564222:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1bedac0.
00010000:00000010:3.1:1787760692.271371:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6800.
00010000:00000010:3.1:1787760692.580712:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0e00.
00010000:00000010:3.1:1787760692.580727:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6c000.
00010000:00000010:3.1:1787760692.580732:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa38a00.
00000080:00000010:3.1:1787760692.592432:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9340.
00000080:00000010:3.1:1787760692.595625:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e6300.
00010000:00000010:3.1:1787760692.624734:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0e00.
00010000:00000010:3.1:1787760692.624748:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922e00.
00000080:00000010:3.1:1787760692.662407:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0caa40.
00010000:00000010:3.1:1787760692.903493:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101500.
00010000:00000010:3.1:1787760692.903508:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888a00.
00000080:00000010:3.1:1787760692.911720:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303640.
00000080:00000010:3.1:1787760692.915432:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c21edac0.
00000080:00000010:3.1:1787760694.671409:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e4a40.
00010000:00000010:3.1:1787760694.705214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0b00.
00010000:00000010:3.1:1787760694.705230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922000.
00000080:00000010:3.1:1787760694.745229:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca720.
00000080:00000010:3.1:1787760694.745246:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e39c0.
00010000:00000010:3.1:1787760694.745252:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0b00.
00010000:00000010:3.1:1787760694.745256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922000.
00000080:00000010:3.1:1787760694.750267:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cab40.
00000080:00000010:3.1:1787760694.750295:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e2940.
00010000:00000010:3.1:1787760694.758411:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0b00.
00010000:00000010:3.1:1787760694.758424:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922400.
00000080:00000010:3.1:1787760695.876595:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303720.
00000080:00000010:3.1:1787760695.876631:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c21ea100.
00010000:00000010:3.1:1787760695.876638:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101500.
00010000:00000010:3.1:1787760695.876643:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888a00.
00010000:00000010:3.1:1787760695.979349:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49d00.
00010000:00000010:3.1:1787760695.979367:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3c00.
00000080:00000010:3.1:1787760696.000769:0:257570:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303be0.
00000080:00000010:3.1:1787760696.000793:0:257570:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18cd280.
00010000:00000010:3.1:1787760696.036918:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798ba00.
00010000:00000010:3.1:1787760696.036937:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101a00.
00010000:00000010:3.1:1787760696.036940:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6de00.
00010000:00000010:3.1:1787760696.036943:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101200.
00010000:00000010:3.1:1787760696.036946:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7889e00.
00010000:00000010:3.1:1787760698.095692:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0200.
00010000:00000010:3.1:1787760698.095699:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923000.
00010000:00000010:3.1:1787760698.252165:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d100.
00010000:00000010:3.1:1787760698.252177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6c800.
00010000:00000010:3.1:1787760698.252181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1c00.
00010000:00000010:3.1:1787760698.252188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2600.
00000080:00000010:3.1:1787760698.408607:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18cb9c0.
00000080:00000010:3.1:1787760698.408627:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dce00.
00010000:00000010:3.1:1787760699.266269:0:257578:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3c00.
00010000:00000010:3.1:1787760699.266286:0:257578:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49600.
00010000:00000010:3.1:1787760699.266329:0:257578:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49d00.
00010000:00000010:3.1:1787760699.266333:0:257578:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2200.
00000080:00000010:3.1:1787760699.318823:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18ca100.
00000080:00000010:3.1:1787760699.318841:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303340.
00010000:00000010:3.1:1787760699.345384:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922c00.
00000080:00000010:3.1:1787760699.464628:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303500.
00000080:00000010:3.1:1787760699.464653:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e0000.
00010000:00000010:3.1:1787760699.860270:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0500.
00010000:00000010:3.1:1787760699.860285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3c00.
00000080:00000010:3.1:1787760699.871378:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93034c0.
00000080:00000010:3.1:1787760699.871392:0:303183:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e0000.
00010000:00000010:3.1:1787760699.877326:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0500.
00010000:00000010:3.1:1787760699.877339:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922a00.
00000080:00000010:3.1:1787760699.891088:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cad20.
00000080:00000010:3.1:1787760699.891332:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e1080.
00010000:00000010:3.1:1787760699.911058:0:257570:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0500.
00010000:00000010:3.1:1787760699.911076:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922a00.
00000080:00000010:3.1:1787760699.924652:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca260.
00000080:00000010:3.1:1787760699.924672:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e1080.
00000080:00000010:3.1:1787760700.860264:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc8c0.
00000080:00000010:3.1:1787760700.860283:0:303183:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1bedac0.
00010000:00000010:3.1:1787760700.874794:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0500.
00010000:00000010:3.1:1787760700.874809:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923e00.
00000080:00000010:3.1:1787760700.890471:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca260.
00010000:00000010:3.1:1787760700.963475:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0500.
00010000:00000010:3.1:1787760700.963491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798ba00.
00000080:00000010:3.1:1787760700.989080:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93037e0.
00000080:00000010:3.1:1787760700.989101:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e0000.
00010000:00000010:3.1:1787760701.014356:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0500.
00010000:00000010:3.1:1787760701.014368:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922a00.
00000080:00000010:3.1:1787760701.038448:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca260.
00000080:00000010:3.1:1787760701.038488:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e0000.
00010000:00000010:3.1:1787760701.064241:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0500.
00010000:00000010:3.1:1787760701.064625:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922a00.
00000080:00000010:3.1:1787760701.068473:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca5a0.
00000080:00000010:3.1:1787760701.068495:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e0000.
00000080:00000010:3.1:1787760701.144301:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1be8840.
00000080:00000010:3.1:1787760702.520055:0:257570:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303d60.
00000080:00000010:3.1:1787760702.520075:0:257570:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1bec200.
00010000:00000010:3.1:1787760702.598471:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0700.
00010000:00000010:3.1:1787760702.598487:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6c200.
00000080:00000010:3.1:1787760702.606457:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9ac0.
00000080:00000010:3.1:1787760702.613257:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e2100.
00010000:00000010:3.1:1787760702.722760:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d000.
00010000:00000010:3.1:1787760702.722780:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6da00.
00010000:00000010:3.1:1787760703.635453:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0700.
00010000:00000010:3.1:1787760703.635470:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798ac00.
00010000:00000010:3.1:1787760703.811379:0:257570:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0700.
00010000:00000010:3.1:1787760703.811387:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6cc00.
00000080:00000010:3.1:1787760703.819525:0:257570:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9540.
00000080:00000010:3.1:1787760704.087638:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e2940.
00000080:00000010:3.1:1787760704.266478:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc840.
00000080:00000010:3.1:1787760704.271371:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1bef380.
00010000:00000010:3.1:1787760704.286703:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0700.
00010000:00000010:3.1:1787760704.286717:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923600.
00000080:00000010:3.1:1787760704.562493:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e2100.
00010000:00000010:3.1:1787760704.975203:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49600.
00010000:00000010:3.1:1787760704.975214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6ca00.
00010000:00000010:3.1:1787760705.047201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3db00.
00010000:00000010:3.1:1787760705.047221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6c600.
00010000:00000010:3.1:1787760705.106496:0:257571:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0700.
00010000:00000010:3.1:1787760705.106502:0:257571:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2a00.
00000080:00000010:3.1:1787760705.110354:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303940.
00010000:00000010:3.1:1787760705.703447:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0700.
00010000:00000010:3.1:1787760705.703454:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6ce00.
00000080:00000010:3.1:1787760705.710170:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9720.
00000080:00000010:3.1:1787760705.710188:0:303183:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e0840.
00010000:00000010:3.1:1787760705.717207:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0700.
00010000:00000010:3.1:1787760705.717223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922800.
00000080:00000010:3.1:1787760705.734254:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca820.
00000080:00000010:3.1:1787760705.883035:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93039e0.
00000080:00000010:3.1:1787760705.883055:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18cd280.
00000080:00000010:3.1:1787760706.328173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc0e0.
00000080:00000010:3.1:1787760706.331250:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1beb180.
00000080:00000010:3.1:1787760706.354413:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cac40.
00000080:00000010:3.1:1787760706.964197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303980.
00000080:00000010:3.1:1787760706.964229:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18cb9c0.
00000080:00000010:3.1:1787760708.367194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303d20.
00000080:00000010:3.1:1787760708.367219:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18cd280.
00010000:00000010:3.1:1787760708.393353:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0300.
00010000:00000010:3.1:1787760708.394064:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923200.
00000080:00000010:3.1:1787760708.415506:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca000.
00000080:00000010:3.1:1787760708.415529:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e2940.
00010000:00000010:3.1:1787760708.730183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0e00.
00010000:00000010:3.1:1787760708.730198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6dc00.
00000080:00000010:3.1:1787760708.741187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9020.
00000080:00000010:3.1:1787760708.744686:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e5280.
00010000:00000010:3.1:1787760708.758639:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0e00.
00010000:00000010:3.1:1787760708.758654:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923400.
00000080:00000010:3.1:1787760708.766277:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca000.
00000080:00000010:3.1:1787760708.770592:0:303183:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e5280.
00000080:00000010:3.1:1787760709.047957:0:257570:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303060.
00000080:00000010:3.1:1787760709.047992:0:257570:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18cdac0.
00010000:00000010:3.1:1787760709.247441:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0e00.
00010000:00000010:3.1:1787760709.247457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798a200.
00000080:00000010:3.1:1787760709.247466:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc800.
00000080:00000010:3.1:1787760709.247483:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e5280.
00010000:00000010:3.1:1787760709.663619:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0e00.
00010000:00000010:3.1:1787760709.663627:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d200.
00010000:00000010:3.1:1787760709.724242:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3db00.
00010000:00000010:3.1:1787760709.724259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6c400.
00000080:00000010:3.1:1787760709.737375:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e7380.
00000080:00000010:3.1:1787760710.078658:0:257570:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93032c0.
00000080:00000010:3.1:1787760710.078679:0:257570:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18cd280.
00010000:00000010:3.1:1787760710.929253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d000.
00010000:00000010:3.1:1787760710.929271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d000.
00000080:00000010:3.1:1787760710.979309:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9d60.
00000080:00000010:3.1:1787760710.979328:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e4a40.
00010000:00000010:3.1:1787760711.795037:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49200.
00010000:00000010:3.1:1787760711.795051:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3c00.
00000080:00000010:3.1:1787760711.808485:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93032c0.
00000080:00000010:3.1:1787760711.808515:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18cdac0.
00010000:00000010:3.1:1787760711.965185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3da00.
00010000:00000010:3.1:1787760711.965196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3a00.
00000080:00000010:3.1:1787760712.020336:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303560.
00000080:00000010:3.1:1787760712.020351:0:303183:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18cb9c0.
00010000:00000010:3.1:1787760712.025587:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0e00.
00010000:00000010:3.1:1787760712.025602:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923a00.
00000080:00000010:3.1:1787760712.065462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9640.
00000080:00000010:3.1:1787760712.065559:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e6b40.
00010000:00000010:3.1:1787760712.084192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0e00.
00010000:00000010:3.1:1787760712.084215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923a00.
00000080:00000010:3.1:1787760712.109391:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca5a0.
00000080:00000010:3.1:1787760712.317447:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e0000.
00010000:00000010:3.1:1787760712.561107:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0e00.
00010000:00000010:3.1:1787760712.561120:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6c400.
00010000:00000010:3.1:1787760712.755239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0e00.
00010000:00000010:3.1:1787760712.755254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6c600.
00010000:00000010:3.1:1787760712.890193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0e00.
00010000:00000010:3.1:1787760712.890209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6de00.
00000080:00000010:3.1:1787760712.905270:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e97e0.
00010000:00000010:3.1:1787760713.692297:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3dc00.
00010000:00000010:3.1:1787760713.692316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6c200.
00010000:00000010:3.1:1787760713.828196:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3df00.
00010000:00000010:3.1:1787760713.828213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d400.
00010000:00000010:3.1:1787760715.118309:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d700.
00010000:00000010:3.1:1787760715.118325:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6c400.
00000080:00000010:3.1:1787760715.296099:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e3180.
00000080:00000010:3.1:1787760715.386421:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303ca0.
00000080:00000010:3.1:1787760715.386447:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18ca100.
00010000:00000010:3.1:1787760715.805303:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d100.
00010000:00000010:3.1:1787760715.805320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6ca00.
00010000:00000010:3.1:1787760716.018560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0e00.
00010000:00000010:3.1:1787760716.018574:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d200.
00000080:00000010:3.1:1787760716.029230:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dce40.
00000080:00000010:3.1:1787760716.029257:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e1080.
00010000:00000010:3.1:1787760716.042183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0e00.
00010000:00000010:3.1:1787760716.042203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923600.
00000080:00000010:3.1:1787760716.080560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cae80.
00000080:00000010:3.1:1787760716.080588:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e1080.
00010000:00000010:3.1:1787760716.104186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0e00.
00010000:00000010:3.1:1787760716.104211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923600.
00000080:00000010:3.1:1787760716.130989:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca220.
00000080:00000010:3.1:1787760716.286166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303de0.
00000080:00000010:3.1:1787760716.286184:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18ca100.
00000080:00000010:3.1:1787760716.475372:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93035e0.
00000080:00000010:3.1:1787760716.475396:0:32:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e2100.
00010000:00000010:3.1:1787760716.475404:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0e00.
00010000:00000010:3.1:1787760716.475409:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923000.
00010000:00000010:3.1:1787760716.626274:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d200.
00010000:00000010:3.1:1787760716.626289:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d200.
00000080:00000010:3.1:1787760716.663286:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e95e0.
00000080:00000010:3.1:1787760716.663316:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e18c0.
00000080:00000010:3.1:1787760716.854373:0:257571:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9640.
00000080:00000010:3.1:1787760716.862307:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e39c0.
00010000:00000010:3.1:1787760716.882585:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0e00.
00010000:00000010:3.1:1787760716.882598:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922e00.
00000080:00000010:3.1:1787760716.890280:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cab40.
00000080:00000010:3.1:1787760716.890305:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e0840.
00010000:00000010:3.1:1787760717.066235:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49200.
00010000:00000010:3.1:1787760717.066245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3800.
00010000:00000010:3.1:1787760717.084488:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d600.
00010000:00000010:3.1:1787760717.084505:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6ca00.
00010000:00000010:3.1:1787760717.164397:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d400.
00010000:00000010:3.1:1787760717.164408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6c600.
00000080:00000010:3.1:1787760717.325218:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303940.
00000080:00000010:3.1:1787760717.325247:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18cd280.
00010000:00000010:3.1:1787760717.333618:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0f00.
00010000:00000010:3.1:1787760717.333634:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922e00.
00000080:00000010:3.1:1787760717.372962:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303860.
00000080:00000010:3.1:1787760717.372983:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e0840.
00000080:00000010:3.1:1787760717.533202:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1bee300.
00010000:00000010:3.1:1787760717.542637:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d300.
00010000:00000010:3.1:1787760717.542651:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922a00.
00010000:00000010:3.1:1787760717.576278:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0d00.
00010000:00000010:3.1:1787760717.576293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2c00.
00000080:00000010:3.1:1787760717.586187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303280.
00000080:00000010:3.1:1787760719.377521:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc00.
00000080:00000010:3.1:1787760719.377550:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1bea100.
00010000:00000010:3.1:1787760719.386962:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0d00.
00010000:00000010:3.1:1787760719.386985:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923e00.
00000080:00000010:3.1:1787760719.396331:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca4e0.
00010000:00000010:3.1:1787760719.662982:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0d00.
00010000:00000010:3.1:1787760719.662992:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798a000.
00000080:00000010:3.1:1787760719.662999:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc280.
00000080:00000010:3.1:1787760719.663014:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e0840.
00010000:00000010:3.1:1787760720.358184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0d00.
00010000:00000010:3.1:1787760720.358199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d000.
00000080:00000010:3.1:1787760720.367777:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9000.
00000080:00000010:3.1:1787760720.367804:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e0840.
00010000:00000010:3.1:1787760720.530222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0d00.
00010000:00000010:3.1:1787760720.530236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798a800.
00000080:00000010:3.1:1787760720.566200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc940.
00000080:00000010:3.1:1787760720.566228:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e0840.
00010000:00000010:3.1:1787760720.575575:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0d00.
00010000:00000010:3.1:1787760720.575586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923e00.
00000080:00000010:3.1:1787760720.594174:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca280.
00000080:00000010:3.1:1787760720.599720:0:303183:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e0000.
00010000:00000010:3.1:1787760720.607688:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0d00.
00010000:00000010:3.1:1787760720.607713:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923e00.
00000080:00000010:3.1:1787760721.520380:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18cb9c0.
00010000:00000010:3.1:1787760721.992353:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0d00.
00010000:00000010:3.1:1787760721.992362:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798a400.
00000080:00000010:3.1:1787760722.015537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc0a0.
00000080:00000010:3.1:1787760722.015566:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e0000.
00000080:00000010:3.1:1787760722.236124:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18cdac0.
00010000:00000010:3.1:1787760722.637521:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0000.
00010000:00000010:3.1:1787760722.637579:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b000.
00000080:00000010:3.1:1787760722.649184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc2c0.
00000080:00000010:3.1:1787760722.649209:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e6b40.
00000080:00000010:3.1:1787760722.918197:0:257571:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e4a40.
00010000:00000010:3.1:1787760722.930725:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0c00.
00010000:00000010:3.1:1787760722.930741:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922e00.
00010000:00000010:3.1:1787760723.087264:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0c00.
00010000:00000010:3.1:1787760723.087280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d200.
00000080:00000010:3.1:1787760723.094699:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e95c0.
00000080:00000010:3.1:1787760723.098198:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e2940.
00000080:00000010:3.1:1787760723.257241:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc9e0.
00000080:00000010:3.1:1787760723.257272:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1be98c0.
00000080:00000010:3.1:1787760723.317104:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcf20.
00000080:00000010:3.1:1787760723.321969:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1bedac0.
00000080:00000010:3.1:1787760723.532321:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93034c0.
00000080:00000010:3.1:1787760723.537235:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e5280.
00010000:00000010:3.1:1787760723.551709:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0b00.
00010000:00000010:3.1:1787760723.551732:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922000.
00000080:00000010:3.1:1787760723.625211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0caa80.
00000080:00000010:3.1:1787760723.628375:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e2940.
00000080:00000010:3.1:1787760723.734238:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303f40.
00000080:00000010:3.1:1787760723.738298:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18cb9c0.
00010000:00000010:3.1:1787760723.764185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0b00.
00010000:00000010:3.1:1787760723.764199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923000.
00000080:00000010:3.1:1787760723.807711:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0caa80.
00000080:00000010:3.1:1787760723.807737:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e2940.
00010000:00000010:3.1:1787760723.825517:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0b00.
00010000:00000010:3.1:1787760723.825531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923000.
00000080:00000010:3.1:1787760723.846264:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca1e0.
00000080:00000010:3.1:1787760723.849389:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e2940.
00010000:00000010:3.1:1787760723.862279:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0b00.
00010000:00000010:3.1:1787760723.862292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923000.
00000080:00000010:3.1:1787760723.872304:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0caa80.
00010000:00000010:3.1:1787760724.510303:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d300.
00010000:00000010:3.1:1787760724.510320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d600.
00000080:00000010:3.1:1787760724.674815:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303f60.
00000080:00000010:3.1:1787760724.721537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303e60.
00000080:00000010:3.1:1787760724.721566:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18cdac0.
00010000:00000010:3.1:1787760725.658148:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0400.
00010000:00000010:3.1:1787760725.658162:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2800.
00000080:00000010:3.1:1787760725.682182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303a20.
00000080:00000010:3.1:1787760725.687185:0:303183:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e5280.
00010000:00000010:3.1:1787760725.707194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0400.
00010000:00000010:3.1:1787760725.707201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923600.
00000080:00000010:3.1:1787760725.707205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca1e0.
00000080:00000010:3.1:1787760725.707216:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e6300.
00000080:00000010:3.1:1787760725.953364:0:257570:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1beeb40.
00010000:00000010:3.1:1787760727.052791:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49100.
00010000:00000010:3.1:1787760727.052809:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2c00.
00000080:00000010:3.1:1787760727.064227:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18ca100.
00000080:00000010:3.1:1787760727.132089:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e3180.
00000080:00000010:3.1:1787760727.132108:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cac80.
00010000:00000010:3.1:1787760728.823475:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923200.
00010000:00000010:3.1:1787760729.043788:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b200.
00000080:00000010:3.1:1787760729.191473:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303500.
00000080:00000010:3.1:1787760729.733183:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc2e0.
00000080:00000010:3.1:1787760729.733202:0:303183:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1bedac0.
00010000:00000010:3.1:1787760730.250090:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798a600.
00010000:00000010:3.1:1787760732.277548:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922600.
00010000:00000010:3.1:1787760732.757504:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6dc00.
00010000:00000010:3.1:1787760733.625374:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923000.
00010000:00000010:3.1:1787760735.942520:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1e29e00.
00010000:00000010:3.1:1787760735.942534:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104ce00.
00010000:00000010:3.1:1787760735.942538:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19600.
00010000:00000010:3.1:1787760735.942541:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c600.
00000080:00000010:3.1:1787760735.942546:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811da20.
00000080:00000010:3.1:1787760735.942554:0:303183:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1010000.
00010000:00000010:3.1:1787760735.949966:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1e29800.
00010000:00000010:3.1:1787760735.956457:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1e29c00.
00010000:00000010:3.1:1787760735.956469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923e00.
00000080:00000010:3.1:1787760735.960463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca060.
00000080:00000010:3.1:1787760735.960482:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1016b40.
00010000:00000010:3.1:1787760735.960493:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104d200.
00010000:00000010:3.1:1787760735.960498:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19a00.
00010000:00000010:3.1:1787760735.978319:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c000.
00010000:00000010:3.1:1787760735.978328:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19b00.
00010000:00000010:3.1:1787760735.978332:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104cc00.
00000080:00000010:3.1:1787760736.034269:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e0000.
00000080:00000010:3.1:1787760736.034287:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d800.
00010000:00000010:3.1:1787760736.037288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e400.
00010000:00000010:3.1:1787760736.037300:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5200.
00010000:00000010:0.1:1787760736.292626:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f200.
00000080:00000010:0.1:1787760736.305010:0:257576:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303e20.
00000080:00000010:0.1:1787760736.305024:0:257576:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1075ac0.
00010000:00000010:0.1:1787760736.311330:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d200.
00010000:00000010:0.1:1787760736.311336:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5400.
00010000:00000010:0.1:1787760736.333519:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d500.
00010000:00000010:0.1:1787760736.333531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e000.
00010000:00000010:0.1:1787760736.363192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5f00.
00010000:00000010:0.1:1787760736.363209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00010000:00000010:0.1:1787760736.454949:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d600.
00010000:00000010:0.1:1787760736.454964:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d900.
00010000:00000010:0.1:1787760736.454969:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f59200.
00010000:00000010:0.1:1787760736.454974:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5800.
00010000:00000010:0.1:1787760736.454978:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58000.
00000080:00000010:0.1:1787760736.499019:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c40a2100.
00000080:00000010:0.1:1787760736.499034:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93030a0.
00010000:00000010:0.1:1787760736.520353:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5900.
00010000:00000010:0.1:1787760736.522190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104ca00.
00010000:00000010:0.1:1787760736.598069:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35dc00.
00010000:00000010:0.1:1787760736.598083:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f59600.
00010000:00000010:0.1:1787760736.639163:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5b00.
00010000:00000010:0.1:1787760736.639175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c600.
00000080:00000010:3.1:1787760736.716434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d1a0.
00000080:00000010:3.1:1787760736.716454:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e10139c0.
00010000:00000010:0.1:1787760736.738632:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f59c00.
00010000:00000010:0.1:1787760736.738645:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35df00.
00010000:00000010:0.1:1787760736.738650:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d000.
00010000:00000010:0.1:1787760736.738653:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922e00.
00010000:00000010:0.1:1787760736.738656:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35d900.
00010000:00000010:0.1:1787760736.738658:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f59000.
00000080:00000010:0.1:1787760736.782991:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d8ea6300.
00000080:00000010:0.1:1787760736.783012:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dce20.
00010000:00000010:0.1:1787760736.809855:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e600.
00010000:00000010:0.1:1787760736.809873:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35d800.
00010000:00000010:0.1:1787760736.809878:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f59800.
00010000:00000010:0.1:1787760736.809882:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d100.
00010000:00000010:0.1:1787760736.809886:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6da00.
00000080:00000010:0.1:1787760736.841898:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d8ea5280.
00000080:00000010:0.1:1787760736.841919:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93039e0.
00010000:00000010:0.1:1787760736.869723:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5900.
00010000:00000010:0.1:1787760736.869733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f200.
00010000:00000010:0.1:1787760736.869737:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5a00.
00010000:00000010:0.1:1787760736.869741:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f400.
00000080:00000010:3.1:1787760736.875246:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303140.
00000080:00000010:3.1:1787760736.875272:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1076300.
00010000:00000010:3.1:1787760736.910405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58600.
00010000:00000010:3.1:1787760736.910414:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35da00.
00010000:00000010:3.1:1787760736.910417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec815c00.
00010000:00000010:3.1:1787760736.930209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c000.
00010000:00000010:0.1:1787760736.930868:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d400.
00010000:00000010:0.1:1787760736.930878:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923e00.
00000080:00000010:3.1:1787760736.978421:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e4200.
00000080:00000010:3.1:1787760736.978434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303380.
00010000:00000010:3.1:1787760737.000422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e800.
00010000:00000010:3.1:1787760737.000438:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35d700.
00010000:00000010:3.1:1787760737.000442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814600.
00010000:00000010:3.1:1787760737.000445:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5f00.
00010000:00000010:3.1:1787760737.000449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec815a00.
00000080:00000010:3.1:1787760737.033413:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d8ea4200.
00000080:00000010:3.1:1787760737.033430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303880.
00010000:00000010:0.1:1787760737.045272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec815000.
00010000:00000010:0.1:1787760737.045289:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5400.
00010000:00000010:0.1:1787760737.045293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec815600.
00010000:00000010:0.1:1787760737.053690:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5e00.
00010000:00000010:0.1:1787760737.053704:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec815200.
00000080:00000010:0.1:1787760737.107203:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d8ea18c0.
00000080:00000010:0.1:1787760737.107221:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93030c0.
00010000:00000010:0.1:1787760737.123576:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5c00.
00010000:00000010:0.1:1787760737.123582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f600.
00010000:00000010:0.1:1787760737.123585:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5900.
00010000:00000010:0.1:1787760737.123588:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814e00.
00000080:00000010:0.1:1787760737.136181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303a00.
00000080:00000010:0.1:1787760737.136203:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1077380.
00010000:00000010:0.1:1787760737.156182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec815e00.
00010000:00000010:0.1:1787760737.156197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5b00.
00010000:00000010:0.1:1787760737.156201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec815400.
00010000:00000010:0.1:1787760737.156204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5e00.
00010000:00000010:0.1:1787760737.156394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e200.
00000080:00000010:0.1:1787760737.197382:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d8ea5ac0.
00000080:00000010:0.1:1787760737.197400:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303080.
00010000:00000010:0.1:1787760737.209865:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5900.
00010000:00000010:0.1:1787760737.209880:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539fc00.
00000080:00000010:0.1:1787760737.240285:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303c40.
00000080:00000010:0.1:1787760737.240300:0:303183:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1073180.
00010000:00000010:0.1:1787760737.264746:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814200.
00010000:00000010:0.1:1787760737.264754:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5000.
00010000:00000010:0.1:1787760737.264759:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e400.
00010000:00000010:0.1:1787760737.264762:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5500.
00010000:00000010:0.1:1787760737.264764:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8a00.
00010000:00000010:0.1:1787760737.329283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814800.
00010000:00000010:0.1:1787760737.364287:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0300.
00010000:00000010:0.1:1787760737.364293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846cc00.
00010000:00000010:0.1:1787760737.396310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814400.
00010000:00000010:0.1:1787760737.396325:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5600.
00010000:00000010:0.1:1787760737.396329:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56a00.
00010000:00000010:0.1:1787760737.396332:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5800.
00010000:00000010:0.1:1787760737.396335:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed9a00.
00000080:00000010:0.1:1787760737.437195:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c2256300.
00000080:00000010:0.1:1787760737.437213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303260.
00010000:00000010:0.1:1787760737.442272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed9800.
00010000:00000010:0.1:1787760737.442288:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5100.
00010000:00000010:0.1:1787760737.442292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8200.
00010000:00000010:0.1:1787760737.442295:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9900.
00010000:00000010:0.1:1787760737.442298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed9000.
00000080:00000010:0.1:1787760737.484202:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1366300.
00000080:00000010:0.1:1787760737.484215:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcd20.
00010000:00000010:0.1:1787760737.608549:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9500.
00010000:00000010:0.1:1787760737.608561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8800.
00010000:00000010:0.1:1787760737.678555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f57400.
00010000:00000010:0.1:1787760737.678570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5000.
00010000:00000010:0.1:1787760737.678574:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56c00.
00010000:00000010:0.1:1787760737.678577:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9c00.
00010000:00000010:0.1:1787760737.678580:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed9200.
00000080:00000010:0.1:1787760737.710473:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c2256b40.
00000080:00000010:0.1:1787760737.710492:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303280.
00010000:00000010:0.1:1787760737.710507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56000.
00010000:00000010:0.1:1787760737.710512:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9400.
00010000:00000010:3.1:1787760737.721959:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed9e00.
00010000:00000010:3.1:1787760737.721967:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9e00.
00010000:00000010:3.1:1787760737.721970:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed9c00.
00000080:00000010:3.1:1787760737.777186:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d8cbf380.
00010000:00000010:0.1:1787760737.786437:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19500.
00010000:00000010:0.1:1787760737.786444:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3db00.
00010000:00000010:0.1:1787760737.786449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923a00.
00000080:00000010:0.1:1787760737.798187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d500.
00000080:00000010:0.1:1787760737.798208:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e1080.
00010000:00000010:0.1:1787760737.809477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56e00.
00010000:00000010:0.1:1787760737.809486:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975202160400.
00010000:00000010:0.1:1787760737.809489:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56400.
00010000:00000010:0.1:1787760737.809492:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101f00.
00010000:00000010:0.1:1787760737.809496:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440e00.
00000080:00000010:0.1:1787760737.843191:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c2251080.
00000080:00000010:0.1:1787760737.843210:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303c80.
00010000:00000010:0.1:1787760737.857187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5900.
00010000:00000010:0.1:1787760737.861248:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9600.
00010000:00000010:0.1:1787760737.861263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8e00.
00000080:00000010:0.1:1787760737.884360:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93038a0.
00000080:00000010:0.1:1787760737.884382:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d8cbca40.
00010000:00000010:0.1:1787760737.897978:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f57800.
00010000:00000010:0.1:1787760737.897994:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101b00.
00010000:00000010:0.1:1787760737.897998:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440400.
00010000:00000010:0.1:1787760737.898001:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101d00.
00010000:00000010:0.1:1787760737.898003:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440800.
00010000:00000010:0.1:1787760737.928179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9d00.
00010000:00000010:0.1:1787760737.928195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440c00.
00000080:00000010:0.1:1787760738.094236:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d8cbc200.
00000080:00000010:0.1:1787760738.094253:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcce0.
00000080:00000010:1.1:1787760738.151484:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303900.
00000080:00000010:1.1:1787760738.151508:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c2250840.
00010000:00000010:1.1:1787760738.156337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06800.
00010000:00000010:1.1:1787760738.156352:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101e00.
00010000:00000010:1.1:1787760738.202213:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101200.
00010000:00000010:1.1:1787760738.202225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468400.
00010000:00000010:0.1:1787760738.277478:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104de00.
00010000:00000010:0.1:1787760738.291209:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6800.
00010000:00000010:0.1:1787760738.291224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468e00.
00010000:00000010:0.1:1787760738.291229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6c00.
00010000:00000010:0.1:1787760738.291232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06c00.
00000080:00000010:0.1:1787760738.305213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811df00.
00000080:00000010:0.1:1787760738.305235:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c2252100.
00010000:00000010:0.1:1787760738.321200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad07400.
00010000:00000010:0.1:1787760738.321210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6500.
00010000:00000010:1.1:1787760738.325558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe469600.
00010000:00000010:1.1:1787760738.325569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6d00.
00010000:00000010:1.1:1787760738.325573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06200.
00010000:00000010:0.1:1787760738.338184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ea00.
00010000:00000010:0.1:1787760738.338224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101a00.
00010000:00000010:0.1:1787760738.338228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe469a00.
00000080:00000010:1.1:1787760738.408418:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c2252940.
00000080:00000010:1.1:1787760738.408429:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303440.
00010000:00000010:1.1:1787760738.455206:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101100.
00010000:00000010:1.1:1787760738.455227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe469200.
00010000:00000010:1.1:1787760738.490539:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6a00.
00010000:00000010:1.1:1787760738.490552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06000.
00010000:00000010:0.1:1787760738.592192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6c400.
00010000:00000010:0.1:1787760738.592210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101700.
00010000:00000010:0.1:1787760738.592214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468c00.
00010000:00000010:0.1:1787760738.602213:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101400.
00010000:00000010:0.1:1787760738.602226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe469800.
00010000:00000010:0.1:1787760738.632548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846cc00.
00010000:00000010:1.1:1787760738.649180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6600.
00010000:00000010:1.1:1787760738.649190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06600.
00000080:00000010:0.1:1787760738.664877:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1ae5ac0.
00000080:00000010:0.1:1787760738.664896:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811dec0.
00010000:00000010:0.1:1787760738.679171:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5900.
00010000:00000010:0.1:1787760738.687619:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ea00.
00010000:00000010:0.1:1787760738.687635:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d000.
00010000:00000010:0.1:1787760738.687642:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6ce00.
00000080:00000010:0.1:1787760738.691194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93038a0.
00000080:00000010:0.1:1787760738.691217:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e39c0.
00010000:00000010:0.1:1787760738.770326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923600.
00010000:00000010:0.1:1787760738.770339:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1800.
00010000:00000010:0.1:1787760738.770345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe469c00.
00010000:00000010:0.1:1787760738.770349:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1000.
00010000:00000010:0.1:1787760738.770352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468800.
00010000:00000010:0.1:1787760738.777223:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1100.
00010000:00000010:0.1:1787760738.777235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468a00.
00000080:00000010:0.1:1787760738.830063:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1ae18c0.
00000080:00000010:0.1:1787760738.830082:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcf80.
00010000:00000010:0.1:1787760738.840385:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0d00.
00010000:00000010:0.1:1787760738.840402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ea00.
00010000:00000010:1.1:1787760738.840460:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5900.
00010000:00000010:1.1:1787760738.840467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d200.
00000080:00000010:1.1:1787760738.848132:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93038a0.
00000080:00000010:1.1:1787760738.848165:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e4200.
00010000:00000010:1.1:1787760738.868200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe469e00.
00010000:00000010:1.1:1787760738.868217:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1d00.
00010000:00000010:1.1:1787760738.868221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe469400.
00010000:00000010:1.1:1787760738.868225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d000.
00010000:00000010:1.1:1787760738.868228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3de00.
00010000:00000010:1.1:1787760738.868233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d000.
00010000:00000010:1.1:1787760738.890407:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1900.
00010000:00000010:1.1:1787760738.890423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468000.
00000080:00000010:1.1:1787760738.911241:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1ae0840.
00000080:00000010:1.1:1787760738.911255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc580.
00010000:00000010:1.1:1787760738.911266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06400.
00010000:00000010:1.1:1787760738.911273:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6300.
00010000:00000010:1.1:1787760738.932343:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1f00.
00010000:00000010:1.1:1787760738.932356:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2a00.
00010000:00000010:0.1:1787760738.942514:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1e00.
00010000:00000010:0.1:1787760738.942521:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798ae00.
00010000:00000010:0.1:1787760738.979182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c800.
00010000:00000010:3.1:1787760738.991394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888c00.
00010000:00000010:3.1:1787760738.991406:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1a00.
00010000:00000010:0.1:1787760739.014945:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad07000.
00010000:00000010:0.1:1787760739.014961:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1500.
00010000:00000010:0.1:1787760739.014965:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7889a00.
00010000:00000010:0.1:1787760739.018332:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1200.
00010000:00000010:0.1:1787760739.018345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888600.
00000080:00000010:1.1:1787760739.053730:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1ae4a40.
00000080:00000010:1.1:1787760739.053751:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc820.
00010000:00000010:1.1:1787760739.065548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3dd00.
00010000:00000010:1.1:1787760739.065564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922c00.
00010000:00000010:0.1:1787760739.065762:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922800.
00010000:00000010:0.1:1787760739.065779:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1700.
00010000:00000010:0.1:1787760739.065783:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888a00.
00000080:00000010:0.1:1787760739.077620:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc580.
00000080:00000010:0.1:1787760739.077645:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e18c0.
00010000:00000010:0.1:1787760739.082238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7889600.
00010000:00000010:0.1:1787760739.082251:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9700.
00010000:00000010:0.1:1787760739.089578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7889800.
00010000:00000010:0.1:1787760739.089604:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1c00.
00010000:00000010:0.1:1787760739.089608:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7889e00.
00010000:00000010:0.1:1787760739.098237:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9800.
00010000:00000010:0.1:1787760739.098252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888e00.
00000080:00000010:0.1:1787760739.145971:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1ae6b40.
00000080:00000010:0.1:1787760739.145991:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc9c0.
00010000:00000010:1.1:1787760739.157632:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0800.
00010000:00000010:1.1:1787760739.157647:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ea00.
00010000:00000010:0.1:1787760739.162488:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923c00.
00010000:00000010:0.1:1787760739.162495:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9100.
00010000:00000010:0.1:1787760739.162499:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888200.
00000080:00000010:0.1:1787760739.167435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303a60.
00000080:00000010:0.1:1787760739.167458:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e2100.
00010000:00000010:1.1:1787760739.182507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798a400.
00010000:00000010:1.1:1787760739.182522:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9b00.
00010000:00000010:0.1:1787760739.205296:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6000.
00010000:00000010:0.1:1787760739.205304:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3600.
00010000:00000010:1.1:1787760739.205559:0:257576:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888000.
00010000:00000010:1.1:1787760739.205566:0:257576:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9c00.
00010000:00000010:1.1:1787760739.205570:0:257576:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7889200.
00000080:00000010:1.1:1787760739.246035:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c21ed280.
00000080:00000010:1.1:1787760739.246054:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcf60.
00010000:00000010:1.1:1787760739.246067:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2600.
00010000:00000010:1.1:1787760739.246073:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6e00.
00010000:00000010:0.1:1787760739.260423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798ba00.
00010000:00000010:0.1:1787760739.260432:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6100.
00010000:00000010:0.1:1787760739.260436:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2200.
00010000:00000010:1.1:1787760739.269264:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3da00.
00010000:00000010:1.1:1787760739.269280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6de00.
00000080:00000010:0.1:1787760739.332480:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d115d280.
00000080:00000010:0.1:1787760739.332498:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303780.
00010000:00000010:0.1:1787760739.343177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5500.
00010000:00000010:1.1:1787760739.343177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ec00.
00010000:00000010:0.1:1787760739.343187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9900.
00010000:00000010:0.1:1787760739.343190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b200.
00000080:00000010:1.1:1787760739.369332:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303a60.
00000080:00000010:1.1:1787760739.369353:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c21ef380.
00010000:00000010:1.1:1787760739.387262:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0c00.
00010000:00000010:1.1:1787760739.387274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923200.
00000080:00000010:3.1:1787760739.409182:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e6b40.
00000080:00000010:3.1:1787760739.409200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cad00.
00010000:00000010:3.1:1787760739.423464:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0000.
00010000:00000010:3.1:1787760739.423476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922400.
00010000:00000010:3.1:1787760739.423481:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9400.
00010000:00000010:3.1:1787760739.423484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798a200.
00000080:00000010:3.1:1787760739.443525:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0caae0.
00000080:00000010:3.1:1787760739.443550:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e0840.
00010000:00000010:3.1:1787760739.443563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b000.
00010000:00000010:3.1:1787760739.443570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9f00.
00010000:00000010:0.1:1787760739.463191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798a800.
00010000:00000010:0.1:1787760739.463209:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9a00.
00010000:00000010:0.1:1787760739.463213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798a000.
00010000:00000010:1.1:1787760739.484400:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9300.
00010000:00000010:1.1:1787760739.484408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798aa00.
00000080:00000010:0.1:1787760739.531533:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c21ea100.
00000080:00000010:0.1:1787760739.531554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc80.
00010000:00000010:0.1:1787760739.548156:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6c800.
00010000:00000010:0.1:1787760739.548162:0:13:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9200.
00010000:00000010:0.1:1787760739.548166:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b800.
00010000:00000010:1.1:1787760739.553876:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0100.
00010000:00000010:1.1:1787760739.553892:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f800.
00000080:00000010:1.1:1787760739.559163:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303780.
00000080:00000010:1.1:1787760739.559192:0:303183:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e12e2940.
00010000:00000010:1.1:1787760739.639409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104ca00.
00010000:00000010:0.1:1787760739.642325:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9600.
00010000:00000010:0.1:1787760739.642338:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2c00.
00010000:00000010:0.1:1787760739.642342:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6b00.
00010000:00000010:0.1:1787760739.642345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3e00.
00000080:00000010:0.1:1787760739.659090:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d520.
00000080:00000010:0.1:1787760739.659115:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c21ea940.
00010000:00000010:0.1:1787760739.681307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2400.
00010000:00000010:0.1:1787760739.681319:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49e00.
00010000:00000010:0.1:1787760739.685203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798be00.
00010000:00000010:0.1:1787760739.685224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49600.
00010000:00000010:0.1:1787760739.685228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3200.
00010000:00000010:0.1:1787760739.699213:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3df00.
00010000:00000010:0.1:1787760739.699227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6c200.
00010000:00000010:1.1:1787760739.763186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9e00.
00010000:00000010:1.1:1787760739.763205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104cc00.
00010000:00000010:3.1:1787760739.798546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3000.
00010000:00000010:3.1:1787760739.798560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0600.
00010000:00000010:3.1:1787760739.798565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923400.
00010000:00000010:3.1:1787760739.798569:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49d00.
00010000:00000010:3.1:1787760739.798572:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2000.
00010000:00000010:0.1:1787760739.842182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1e29800.
00010000:00000010:0.1:1787760739.842195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6ca00.
00000080:00000010:0.1:1787760739.852187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cad00.
00000080:00000010:0.1:1787760739.852217:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c96e2100.
00010000:00000010:0.1:1787760739.863197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b400.
00010000:00000010:0.1:1787760739.863216:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49200.
00010000:00000010:0.1:1787760739.863221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3800.
00010000:00000010:0.1:1787760739.863223:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49c00.
00010000:00000010:0.1:1787760739.863225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3a00.
00010000:00000010:1.1:1787760739.875184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49100.
00010000:00000010:1.1:1787760739.875193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2e00.
00000080:00000010:0.1:1787760739.904186:0:303183:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d115b180.
00000080:00000010:0.1:1787760739.904199:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303f40.
00010000:00000010:0.1:1787760739.936336:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5600.
00010000:00000010:0.1:1787760739.936345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ea00.
00010000:00000010:0.1:1787760739.956376:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5600.
00010000:00000010:0.1:1787760739.963800:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f400.
00010000:00000010:0.1:1787760739.963807:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6de00.
00010000:00000010:0.1:1787760739.963812:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441e00.
00010000:00000010:0.1:1787760739.963815:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4fc00.
00000080:00000010:0.1:1787760739.968195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303280.
00000080:00000010:0.1:1787760739.968222:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18b8000.
00010000:00000010:0.1:1787760739.976410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4fa00.
00010000:00000010:0.1:1787760739.976426:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d700.
00010000:00000010:0.1:1787760739.976430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441200.
00000080:00000010:0.1:1787760739.989253:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93035c0.
00000080:00000010:0.1:1787760739.989270:0:303183:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18beb40.
00000080:00000010:0.1:1787760739.989277:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303f40.
00010000:00000010:1.1:1787760740.030948:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5500.
00010000:00000010:1.1:1787760740.030958:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f800.
00010000:00000010:0.1:1787760740.076449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e800.
00010000:00000010:1.1:1787760740.135454:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1ff00.
00010000:00000010:1.1:1787760740.135462:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d000.
00010000:00000010:1.1:1787760740.135465:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5600.
00010000:00000010:1.1:1787760740.135467:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f400.
00000080:00000010:1.1:1787760740.146437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcf60.
00000080:00000010:1.1:1787760740.146465:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1074a40.
00010000:00000010:1.1:1787760740.156452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e800.
00010000:00000010:1.1:1787760740.156463:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19d00.
00010000:00000010:1.1:1787760740.156467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104ce00.
00010000:00000010:1.1:1787760740.156470:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19100.
00010000:00000010:1.1:1787760740.156473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c000.
00010000:00000010:1.1:1787760740.169224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19e00.
00010000:00000010:1.1:1787760740.169234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922400.
00000080:00000010:1.1:1787760740.178313:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18be300.
00000080:00000010:1.1:1787760740.178333:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d7c0.
00010000:00000010:1.1:1787760740.211189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c800.
00010000:00000010:0.1:1787760740.240431:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f300.
00010000:00000010:0.1:1787760740.240440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c600.
00010000:00000010:0.1:1787760740.240443:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1ff00.
00010000:00000010:0.1:1787760740.240446:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d000.
00000080:00000010:1.1:1787760740.240806:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1074a40.
00000080:00000010:1.1:1787760740.240816:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcf60.
00010000:00000010:0.1:1787760740.278227:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5e00.
00010000:00000010:0.1:1787760740.278235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ee00.
00010000:00000010:0.1:1787760740.282476:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5400.
00010000:00000010:0.1:1787760740.294194:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5400.
00010000:00000010:0.1:1787760740.294202:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f200.
00010000:00000010:0.1:1787760740.294207:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846da00.
00000080:00000010:0.1:1787760740.307338:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93035c0.
00000080:00000010:0.1:1787760740.307355:0:303183:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c40a2940.
00010000:00000010:1.1:1787760740.325581:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5400.
00010000:00000010:1.1:1787760740.325596:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f200.
00000080:00000010:1.1:1787760740.335309:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303940.
00000080:00000010:1.1:1787760740.339724:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c40a7380.
00010000:00000010:1.1:1787760740.348579:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f900.
00010000:00000010:1.1:1787760740.348594:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00000080:00000010:1.1:1787760740.358711:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcf60.
00000080:00000010:1.1:1787760740.358735:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1074a40.
00000080:00000010:0.1:1787760740.387237:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811dde0.
00000080:00000010:0.1:1787760740.387256:0:303183:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18b98c0.
00010000:00000010:0.1:1787760740.400506:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5e00.
00010000:00000010:0.1:1787760740.400523:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ee00.
00000080:00000010:1.1:1787760740.414478:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93035c0.
00000080:00000010:1.1:1787760740.414499:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c40a2940.
00010000:00000010:1.1:1787760740.439020:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f900.
00010000:00000010:1.1:1787760740.439031:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00000080:00000010:1.1:1787760740.450406:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcf60.
00000080:00000010:1.1:1787760740.450423:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1074a40.
00000080:00000010:0.1:1787760740.462205:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc820.
00000080:00000010:0.1:1787760740.476214:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18bb9c0.
00010000:00000010:0.1:1787760740.492427:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5e00.
00010000:00000010:0.1:1787760740.492440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ee00.
00000080:00000010:0.1:1787760740.497311:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303c40.
00000080:00000010:0.1:1787760740.497331:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c40a0000.
00010000:00000010:0.1:1787760740.524880:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5e00.
00010000:00000010:0.1:1787760740.524894:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c600.
00000080:00000010:0.1:1787760740.529481:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d920.
00000080:00000010:0.1:1787760740.540902:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c40a0000.
00010000:00000010:1.1:1787760740.546442:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5e00.
00010000:00000010:1.1:1787760740.546457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ee00.
00000080:00000010:1.1:1787760740.546465:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303c40.
00000080:00000010:1.1:1787760740.550195:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1074a40.
00010000:00000010:1.1:1787760740.561726:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f900.
00010000:00000010:1.1:1787760740.561742:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00000080:00000010:1.1:1787760740.568178:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcf60.
00000080:00000010:1.1:1787760740.573266:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1074a40.
00010000:00000010:1.1:1787760740.575294:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f900.
00010000:00000010:1.1:1787760740.575310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00000080:00000010:1.1:1787760740.584319:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc5e0.
00000080:00000010:0.1:1787760740.588202:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1070840.
00010000:00000010:0.1:1787760740.600663:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5800.
00010000:00000010:0.1:1787760740.600672:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e600.
00000080:00000010:0.1:1787760740.607182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303a00.
00000080:00000010:0.1:1787760740.607197:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c40a0000.
00010000:00000010:1.1:1787760740.613572:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5800.
00010000:00000010:1.1:1787760740.613587:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e400.
00010000:00000010:3.1:1787760740.645158:0:31:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1e29e00.
00010000:00000010:3.1:1787760740.646922:0:31:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104de00.
00000080:00000010:3.1:1787760740.652451:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d620.
00000080:00000010:3.1:1787760740.656294:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e10139c0.
00010000:00000010:0.1:1787760740.660853:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5d00.
00010000:00000010:0.1:1787760740.660860:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923000.
00000080:00000010:0.1:1787760740.665493:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cab80.
00000080:00000010:0.1:1787760740.665519:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c40a0000.
00010000:00000010:0.1:1787760740.669369:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5d00.
00010000:00000010:0.1:1787760740.669382:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e600.
00000080:00000010:0.1:1787760740.680343:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303a00.
00000080:00000010:0.1:1787760740.680370:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c40a5ac0.
00010000:00000010:1.1:1787760740.700461:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f900.
00010000:00000010:1.1:1787760740.700470:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e600.
00000080:00000010:1.1:1787760740.723218:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93039e0.
00000080:00000010:1.1:1787760740.723245:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1074a40.
00010000:00000010:1.1:1787760740.730862:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f900.
00010000:00000010:1.1:1787760740.730873:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00000080:00000010:3.1:1787760740.733047:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc5e0.
00000080:00000010:3.1:1787760740.733064:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1072100.
00000080:00000010:1.1:1787760740.760174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0caf00.
00000080:00000010:1.1:1787760740.760192:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e10139c0.
00000080:00000010:1.1:1787760740.760196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc300.
00010000:00000010:0.1:1787760740.771193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d600.
00010000:00000010:0.1:1787760740.771210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fc00.
00010000:00000010:1.1:1787760740.774280:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f400.
00010000:00000010:1.1:1787760740.774297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c800.
00010000:00000010:0.1:1787760740.775219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d800.
00010000:00000010:0.1:1787760740.775228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fa00.
00010000:00000010:0.1:1787760740.775233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f000.
00000080:00000010:0.1:1787760740.802631:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c40a2100.
00000080:00000010:0.1:1787760740.802651:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcf60.
00010000:00000010:1.1:1787760740.870184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f100.
00010000:00000010:1.1:1787760740.870203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c200.
00010000:00000010:1.1:1787760740.935249:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f100.
00010000:00000010:0.1:1787760740.950752:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5e00.
00010000:00000010:0.1:1787760740.950771:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f600.
00010000:00000010:0.1:1787760740.950777:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f900.
00010000:00000010:0.1:1787760740.950781:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ca00.
00000080:00000010:0.1:1787760740.963165:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93030a0.
00000080:00000010:0.1:1787760740.970197:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c40a2100.
00010000:00000010:0.1:1787760740.976701:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4fe00.
00010000:00000010:0.1:1787760740.976708:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5300.
00010000:00000010:1.1:1787760740.982399:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5d00.
00010000:00000010:1.1:1787760740.982427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f000.
00010000:00000010:0.1:1787760741.000861:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f400.
00010000:00000010:0.1:1787760741.000886:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c800.
00010000:00000010:0.1:1787760741.078280:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5e00.
00010000:00000010:0.1:1787760741.078288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f600.
00010000:00000010:1.1:1787760741.156894:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f100.
00010000:00000010:1.1:1787760741.176915:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f500.
00010000:00000010:1.1:1787760741.176923:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846de00.
00010000:00000010:1.1:1787760741.176927:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5300.
00010000:00000010:1.1:1787760741.176930:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4fe00.
00000080:00000010:1.1:1787760741.184186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc8e0.
00000080:00000010:1.1:1787760741.184213:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1072940.
00010000:00000010:1.1:1787760741.184226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104d600.
00010000:00000010:1.1:1787760741.184231:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19c00.
00010000:00000010:1.1:1787760741.188237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c400.
00010000:00000010:1.1:1787760741.188246:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19800.
00010000:00000010:1.1:1787760741.188250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104dc00.
00010000:00000010:1.1:1787760741.188254:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19000.
00010000:00000010:1.1:1787760741.188257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104d000.
00010000:00000010:1.1:1787760741.193821:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19e00.
00010000:00000010:1.1:1787760741.193833:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104da00.
00000080:00000010:1.1:1787760741.215925:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18bb9c0.
00000080:00000010:1.1:1787760741.219495:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811dec0.
00010000:00000010:1.1:1787760741.260303:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fc00.
00010000:00000010:1.1:1787760741.260317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c200.
00010000:00000010:1.1:1787760741.299742:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fc00.
00010000:00000010:1.1:1787760741.318656:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1ff00.
00010000:00000010:1.1:1787760741.318670:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ca00.
00010000:00000010:1.1:1787760741.318674:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fa00.
00010000:00000010:1.1:1787760741.318677:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c800.
00000080:00000010:1.1:1787760741.327442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc860.
00000080:00000010:1.1:1787760741.327462:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1072100.
00010000:00000010:1.1:1787760741.327476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846de00.
00010000:00000010:1.1:1787760741.327481:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f100.
00010000:00000010:0.1:1787760741.332187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f500.
00010000:00000010:0.1:1787760741.332204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c400.
00010000:00000010:0.1:1787760741.346282:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f900.
00010000:00000010:0.1:1787760741.346300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846dc00.
00010000:00000010:0.1:1787760741.427213:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5e00.
00010000:00000010:0.1:1787760741.427227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f600.
00010000:00000010:1.1:1787760741.534063:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f100.
00010000:00000010:1.1:1787760741.572913:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f100.
00010000:00000010:1.1:1787760741.572928:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104d000.
00010000:00000010:1.1:1787760741.572932:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19f00.
00010000:00000010:1.1:1787760741.572934:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104d800.
00000080:00000010:1.1:1787760741.590945:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d200.
00000080:00000010:3.1:1787760741.610196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcf40.
00000080:00000010:3.1:1787760741.610220:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18bb9c0.
00000080:00000010:1.1:1787760741.610291:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1072100.
00010000:00000010:3.1:1787760741.615214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104ce00.
00010000:00000010:3.1:1787760741.615228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19d00.
00010000:00000010:0.1:1787760741.615431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846de00.
00010000:00000010:0.1:1787760741.615437:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19500.
00010000:00000010:0.1:1787760741.615440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c000.
00010000:00000010:1.1:1787760741.628710:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19100.
00010000:00000010:1.1:1787760741.628724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104ca00.
00000080:00000010:0.1:1787760741.696596:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18be300.
00000080:00000010:0.1:1787760741.696616:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d4e0.
00010000:00000010:0.1:1787760741.759290:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5700.
00010000:00000010:0.1:1787760741.759303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e400.
00000080:00000010:0.1:1787760741.888891:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18be300.
00010000:00000010:0.1:1787760741.888914:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4fe00.
00010000:00000010:0.1:1787760741.888922:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5400.
00010000:00000010:0.1:1787760741.905943:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c800.
00010000:00000010:0.1:1787760741.905958:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5600.
00010000:00000010:0.1:1787760741.905963:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e200.
00010000:00000010:1.1:1787760741.921060:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5e00.
00010000:00000010:1.1:1787760741.921076:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f600.
00010000:00000010:1.1:1787760741.965214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5500.
00010000:00000010:1.1:1787760741.965226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f000.
00000080:00000010:0.1:1787760742.034372:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c40a2100.
00000080:00000010:0.1:1787760742.034390:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303fa0.
00010000:00000010:1.1:1787760742.106181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fc00.
00010000:00000010:1.1:1787760742.106194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d600.
00010000:00000010:1.1:1787760742.145371:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4fe00.
00010000:00000010:1.1:1787760742.145389:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5400.
00010000:00000010:1.1:1787760742.145392:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5700.
00010000:00000010:1.1:1787760742.145396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e400.
00010000:00000010:1.1:1787760742.145401:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5600.
00010000:00000010:1.1:1787760742.145403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e200.
00010000:00000010:1.1:1787760742.170679:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fa00.
00010000:00000010:1.1:1787760742.170693:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ca00.
00010000:00000010:1.1:1787760742.192387:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1ff00.
00010000:00000010:1.1:1787760742.192399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d800.
00000080:00000010:1.1:1787760742.224397:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1072100.
00000080:00000010:1.1:1787760742.224414:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303fa0.
00010000:00000010:0.1:1787760742.284193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5100.
00010000:00000010:0.1:1787760742.284209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f200.
00010000:00000010:0.1:1787760742.325289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c600.
00010000:00000010:0.1:1787760742.325301:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19200.
00010000:00000010:0.1:1787760742.325305:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19300.
00010000:00000010:0.1:1787760742.325308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104d600.
00010000:00000010:0.1:1787760742.334210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1ff00.
00010000:00000010:0.1:1787760742.334231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d800.
00010000:00000010:1.1:1787760742.349210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5900.
00010000:00000010:1.1:1787760742.349217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ee00.
00010000:00000010:1.1:1787760742.353673:0:257576:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5200.
00010000:00000010:1.1:1787760742.353681:0:257576:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e000.
00000080:00000010:0.1:1787760742.377519:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c40a2100.
00000080:00000010:0.1:1787760742.377537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811daa0.
00010000:00000010:0.1:1787760742.537220:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5900.
00010000:00000010:0.1:1787760742.613026:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923400.
00010000:00000010:0.1:1787760742.723186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ca00.
00010000:00000010:0.1:1787760742.723199:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fa00.
00010000:00000010:0.1:1787760742.723204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fc00.
00010000:00000010:0.1:1787760742.723207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d600.
00010000:00000010:0.1:1787760742.723235:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19600.
00010000:00000010:0.1:1787760742.723238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104de00.
00010000:00000010:0.1:1787760742.739215:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5900.
00010000:00000010:0.1:1787760742.739223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f800.
00010000:00000010:1.1:1787760742.756871:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5600.
00010000:00000010:1.1:1787760742.756885:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ea00.
00000080:00000010:0.1:1787760742.805289:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c40a2100.
00000080:00000010:0.1:1787760742.805302:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcd60.
00010000:00000010:0.1:1787760742.843432:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5100.
00010000:00000010:0.1:1787760742.843447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f200.
00010000:00000010:0.1:1787760742.864520:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19a00.
00010000:00000010:0.1:1787760742.864533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104cc00.
00010000:00000010:0.1:1787760742.935276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f200.
00010000:00000010:1.1:1787760743.018459:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fc00.
00010000:00000010:1.1:1787760743.018470:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846da00.
00010000:00000010:1.1:1787760743.063373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f800.
00010000:00000010:1.1:1787760743.063383:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5900.
00010000:00000010:1.1:1787760743.074230:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5700.
00010000:00000010:1.1:1787760743.074243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4fc00.
00010000:00000010:0.1:1787760743.078176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c000.
00010000:00000010:0.1:1787760743.111415:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f300.
00010000:00000010:0.1:1787760743.111423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00010000:00000010:0.1:1787760743.120448:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fd00.
00010000:00000010:0.1:1787760743.120456:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d200.
00000080:00000010:0.1:1787760743.162301:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1072100.
00000080:00000010:0.1:1787760743.162314:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303240.
00000080:00000010:1.1:1787760743.166257:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303ce0.
00010000:00000010:1.1:1787760743.166286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ec00.
00010000:00000010:1.1:1787760743.166294:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5100.
00010000:00000010:1.1:1787760743.166297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f200.
00010000:00000010:1.1:1787760743.166301:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19b00.
00010000:00000010:1.1:1787760743.166304:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104d200.
00000080:00000010:0.1:1787760743.178933:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c40a2100.
00000080:00000010:0.1:1787760743.178946:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811de20.
00010000:00000010:0.1:1787760743.218475:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5c00.
00010000:00000010:0.1:1787760743.218485:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4fc00.
00010000:00000010:1.1:1787760743.249365:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f200.
00010000:00000010:1.1:1787760743.249378:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5500.
00010000:00000010:1.1:1787760743.249403:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5e00.
00010000:00000010:1.1:1787760743.249407:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ec00.
00010000:00000010:1.1:1787760743.249411:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5400.
00010000:00000010:1.1:1787760743.249414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4fa00.
00010000:00000010:0.1:1787760743.319464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f800.
00010000:00000010:1.1:1787760743.377066:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fc00.
00010000:00000010:1.1:1787760743.377079:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d000.
00010000:00000010:1.1:1787760743.377082:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4fc00.
00000080:00000010:1.1:1787760743.384294:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc40.
00000080:00000010:1.1:1787760743.384316:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1075ac0.
00000080:00000010:1.1:1787760743.384324:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93032c0.
00010000:00000010:0.1:1787760743.397934:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c600.
00010000:00000010:0.1:1787760743.397950:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5c00.
00010000:00000010:0.1:1787760743.397954:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846da00.
00010000:00000010:0.1:1787760743.397958:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19300.
00010000:00000010:0.1:1787760743.397961:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104d600.
00000080:00000010:0.1:1787760743.412409:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c40a2100.
00000080:00000010:0.1:1787760743.412424:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcb00.
00010000:00000010:1.1:1787760743.448161:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fc00.
00010000:00000010:1.1:1787760743.448169:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d000.
00010000:00000010:1.1:1787760743.482372:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fc00.
00010000:00000010:0.1:1787760743.519289:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5d00.
00010000:00000010:0.1:1787760743.519300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e800.
00010000:00000010:0.1:1787760743.519305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f400.
00000080:00000010:0.1:1787760743.524260:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303bc0.
00000080:00000010:0.1:1787760743.537624:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c40a2100.
00000080:00000010:0.1:1787760743.537641:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d3c0.
00010000:00000010:0.1:1787760743.549230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ea00.
00010000:00000010:1.1:1787760743.566349:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5c00.
00010000:00000010:1.1:1787760743.566360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f800.
00010000:00000010:1.1:1787760743.756421:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1ff00.
00010000:00000010:1.1:1787760743.756434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846da00.
00010000:00000010:0.1:1787760743.789728:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d400.
00010000:00000010:0.1:1787760743.789746:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fc00.
00010000:00000010:0.1:1787760743.792307:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fa00.
00010000:00000010:0.1:1787760743.792316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d000.
00010000:00000010:0.1:1787760743.833250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104d200.
00000080:00000010:0.1:1787760743.881085:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e10718c0.
00000080:00000010:0.1:1787760743.881103:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc1c0.
00010000:00000010:0.1:1787760743.884078:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5d00.
00010000:00000010:0.1:1787760743.884094:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ea00.
00010000:00000010:1.1:1787760743.948192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49500.
00010000:00000010:1.1:1787760743.948207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2800.
00000080:00000010:1.1:1787760743.952203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303240.
00000080:00000010:1.1:1787760743.952232:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c40a2100.
00000080:00000010:0.1:1787760743.952338:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811de60.
00010000:00000010:0.1:1787760743.984199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846da00.
00010000:00000010:0.1:1787760743.984214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19100.
00010000:00000010:0.1:1787760743.984219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846cc00.
00010000:00000010:0.1:1787760743.984222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19200.
00010000:00000010:0.1:1787760743.984225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104d800.
00000080:00000010:0.1:1787760744.022398:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d18bb9c0.
00000080:00000010:0.1:1787760744.022410:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc4e0.
00010000:00000010:0.1:1787760744.077278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c000.
00010000:00000010:1.1:1787760744.168262:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5c00.
00010000:00000010:1.1:1787760744.168269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f400.
00010000:00000010:1.1:1787760744.168273:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5d00.
00010000:00000010:1.1:1787760744.168276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ea00.
00010000:00000010:1.1:1787760744.175357:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5800.
00010000:00000010:1.1:1787760744.175367:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e800.
00010000:00000010:0.1:1787760744.187282:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5300.
00010000:00000010:0.1:1787760744.187295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e000.
00000080:00000010:1.1:1787760744.219347:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c40a2100.
00000080:00000010:1.1:1787760744.219369:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303bc0.
00010000:00000010:1.1:1787760744.256769:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00010000:00000010:0.1:1787760744.352881:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e400.
00010000:00000010:1.1:1787760744.394157:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5300.
00010000:00000010:1.1:1787760744.394680:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e000.
00010000:00000010:1.1:1787760744.394685:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d200.
00010000:00000010:1.1:1787760744.397934:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5400.
00010000:00000010:1.1:1787760744.397941:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ee00.
00010000:00000010:0.1:1787760744.402365:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5e00.
00010000:00000010:0.1:1787760744.402378:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e200.
00000080:00000010:1.1:1787760744.426601:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1ae0840.
00000080:00000010:1.1:1787760744.426616:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93032c0.
00000080:00000010:1.1:1787760744.426624:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcac0.
00010000:00000010:1.1:1787760744.440512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d200.
00010000:00000010:1.1:1787760744.444446:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f100.
00010000:00000010:1.1:1787760744.444457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00010000:00000010:1.1:1787760744.444462:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1ff00.
00010000:00000010:1.1:1787760744.444465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c600.
00000080:00000010:1.1:1787760744.449874:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1071080.
00000080:00000010:1.1:1787760744.449884:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc0e0.
00010000:00000010:1.1:1787760744.471551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00010000:00000010:0.1:1787760744.500469:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5e00.
00010000:00000010:0.1:1787760744.500481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d600.
00010000:00000010:0.1:1787760744.500484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c600.
00000080:00000010:0.1:1787760744.520259:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcac0.
00000080:00000010:0.1:1787760744.525201:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1ae0840.
00010000:00000010:1.1:1787760744.614172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fa00.
00010000:00000010:1.1:1787760744.614184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846dc00.
00010000:00000010:1.1:1787760744.696549:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e200.
00010000:00000010:1.1:1787760744.696559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5e00.
00010000:00000010:0.1:1787760744.698652:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f100.
00010000:00000010:0.1:1787760744.698666:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ca00.
00010000:00000010:1.1:1787760744.705471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c800.
00010000:00000010:0.1:1787760744.770145:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f500.
00010000:00000010:0.1:1787760744.770162:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d800.
00010000:00000010:0.1:1787760744.772216:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f400.
00010000:00000010:0.1:1787760744.772228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846de00.
00000080:00000010:0.1:1787760744.819244:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e10718c0.
00000080:00000010:0.1:1787760744.819327:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303760.
00010000:00000010:1.1:1787760744.870232:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fa00.
00010000:00000010:1.1:1787760744.870245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c800.
00010000:00000010:0.1:1787760745.015465:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5700.
00010000:00000010:0.1:1787760745.015478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f000.
00010000:00000010:0.1:1787760745.025614:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5900.
00010000:00000010:0.1:1787760745.025626:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f600.
00000080:00000010:0.1:1787760745.038961:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1ae0840.
00000080:00000010:0.1:1787760745.038981:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303980.
00000080:00000010:0.1:1787760745.047495:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc140.
00010000:00000010:1.1:1787760745.070307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f600.
00010000:00000010:1.1:1787760745.070323:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1ff00.
00010000:00000010:1.1:1787760745.070328:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00010000:00000010:1.1:1787760745.070332:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f900.
00010000:00000010:1.1:1787760745.070334:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d200.
00000080:00000010:1.1:1787760745.081171:0:303183:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1071080.
00000080:00000010:1.1:1787760745.081181:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303760.
00010000:00000010:1.1:1787760745.114175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c800.
00010000:00000010:1.1:1787760745.234192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5100.
00010000:00000010:1.1:1787760745.234205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e400.
00010000:00000010:1.1:1787760745.293417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f000.
00010000:00000010:1.1:1787760745.293429:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5900.
00010000:00000010:1.1:1787760745.293433:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5700.
00010000:00000010:1.1:1787760745.293436:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e600.
00010000:00000010:0.1:1787760745.303376:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00010000:00000010:0.1:1787760745.321412:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5200.
00010000:00000010:0.1:1787760745.321424:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f600.
00010000:00000010:1.1:1787760745.326635:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5600.
00010000:00000010:1.1:1787760745.326644:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4fe00.
00000080:00000010:1.1:1787760745.363465:0:303183:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c2250840.
00000080:00000010:1.1:1787760745.363478:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303980.
00000080:00000010:0.1:1787760745.369351:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc0e0.
00010000:00000010:1.1:1787760745.382558:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846dc00.
00010000:00000010:1.1:1787760745.382567:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1ff00.
00010000:00000010:1.1:1787760745.382572:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c800.
00010000:00000010:1.1:1787760745.382575:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f900.
00010000:00000010:1.1:1787760745.382577:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d200.
00000080:00000010:1.1:1787760745.393583:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1071080.
00000080:00000010:1.1:1787760745.393597:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dce60.
00010000:00000010:1.1:1787760745.438018:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f300.
00010000:00000010:1.1:1787760745.438029:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c200.
00010000:00000010:0.1:1787760745.485273:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5200.
00010000:00000010:1.1:1787760745.493570:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f300.
00010000:00000010:1.1:1787760745.493578:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c200.
00010000:00000010:1.1:1787760745.493581:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fc00.
00010000:00000010:1.1:1787760745.493584:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846dc00.
00000080:00000010:1.1:1787760745.502427:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dca60.
00000080:00000010:1.1:1787760745.502446:0:303183:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c40a2100.
00010000:00000010:1.1:1787760745.515257:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fc00.
00010000:00000010:1.1:1787760745.515273:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00010000:00000010:1.1:1787760745.515277:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fd00.
00010000:00000010:1.1:1787760745.515279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c000.
00000080:00000010:1.1:1787760745.520126:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc960.
00000080:00000010:1.1:1787760745.520159:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c21ea940.
00010000:00000010:1.1:1787760745.528822:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f300.
00010000:00000010:1.1:1787760745.528830:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fd00.
00010000:00000010:1.1:1787760745.528833:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846dc00.
00000080:00000010:1.1:1787760745.541281:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dca60.
00000080:00000010:1.1:1787760745.541308:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c21ea100.
00010000:00000010:1.1:1787760745.541318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d200.
00010000:00000010:1.1:1787760745.541322:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f900.
00010000:00000010:1.1:1787760745.556174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f600.
00010000:00000010:1.1:1787760745.556185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1ff00.
00010000:00000010:1.1:1787760745.556190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c800.
00010000:00000010:0.1:1787760745.560156:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fa00.
00010000:00000010:0.1:1787760745.560182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c400.
00000080:00000010:1.1:1787760745.591449:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1071080.
00000080:00000010:1.1:1787760745.591468:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc620.
00010000:00000010:1.1:1787760745.650178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fa00.
00010000:00000010:1.1:1787760745.650193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c200.
00010000:00000010:0.1:1787760745.708295:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5200.
00010000:00000010:0.1:1787760745.724231:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5200.
00010000:00000010:0.1:1787760745.724258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ee00.
00010000:00000010:0.1:1787760745.724264:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fc00.
00010000:00000010:0.1:1787760745.724269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ce00.
00000080:00000010:0.1:1787760745.763423:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93030a0.
00000080:00000010:0.1:1787760745.766191:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c2256300.
00010000:00000010:1.1:1787760745.778182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5200.
00010000:00000010:1.1:1787760745.781531:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fa00.
00010000:00000010:1.1:1787760745.781547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ee00.
00000080:00000010:1.1:1787760745.810874:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303f40.
00000080:00000010:1.1:1787760745.810898:0:303183:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c21ed280.
00010000:00000010:1.1:1787760745.818550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fa00.
00010000:00000010:1.1:1787760745.818563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c200.
00010000:00000010:1.1:1787760745.818568:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f300.
00010000:00000010:1.1:1787760745.818572:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c000.
00000080:00000010:1.1:1787760745.840297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc960.
00000080:00000010:1.1:1787760745.840324:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1071080.
00010000:00000010:1.1:1787760745.858277:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f300.
00010000:00000010:0.1:1787760745.933375:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e800.
00010000:00000010:1.1:1787760746.006513:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f300.
00010000:00000010:1.1:1787760746.006527:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d800.
00010000:00000010:1.1:1787760746.006531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c000.
00000080:00000010:1.1:1787760746.012341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc380.
00000080:00000010:1.1:1787760746.012370:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c21ed280.
00010000:00000010:0.1:1787760746.026234:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f300.
00010000:00000010:0.1:1787760746.026251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c000.
00010000:00000010:0.1:1787760746.117190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e200.
00010000:00000010:0.1:1787760746.117201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fd00.
00010000:00000010:0.1:1787760746.117206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846dc00.
00010000:00000010:0.1:1787760746.121986:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1ff00.
00010000:00000010:0.1:1787760746.121995:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c800.
00010000:00000010:0.1:1787760746.138929:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f400.
00010000:00000010:0.1:1787760746.138942:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c200.
00000080:00000010:0.1:1787760746.175291:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c21ea100.
00000080:00000010:0.1:1787760746.175309:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc620.
00010000:00000010:0.1:1787760746.195586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ea00.
00010000:00000010:0.1:1787760746.195604:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19700.
00010000:00000010:0.1:1787760746.195609:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888400.
00010000:00000010:0.1:1787760746.195612:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5e00.
00010000:00000010:0.1:1787760746.195615:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e200.
00000080:00000010:0.1:1787760746.207007:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93030a0.
00000080:00000010:0.1:1787760746.214586:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1be8000.
00010000:00000010:0.1:1787760746.274822:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ea00.
00010000:00000010:0.1:1787760746.274833:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19400.
00010000:00000010:0.1:1787760746.274838:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7889400.
00010000:00000010:0.1:1787760746.274841:0:303183:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19900.
00010000:00000010:0.1:1787760746.274844:0:303183:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104d400.
00000080:00000010:0.1:1787760746.279474:0:303183:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93030a0.
00000080:00000010:0.1:1787760746.279497:0:303183:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751c21eca40.
00010000:00000010:3.1:1787760746.299610:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d800.
00010000:00000010:3.1:1787760746.299626:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d900.
00010000:00000010:3.1:1787760746.299632:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa39c00.
00010000:00000010:3.1:1787760746.305669:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5c00.
00010000:00000010:3.1:1787760746.305678:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4fc00.
00000080:00000010:3.1:1787760746.313194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc380.
00000080:00000010:3.1:1787760746.316194:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1bea940.
00010000:00000010:1.1:1787760749.557881:0:1365:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5400.
00010000:00000010:1.1:1787760749.557891:0:1365:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f400.
00010000:00000010:1.1:1787760749.560173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1e29300.
00010000:00000010:1.1:1787760749.560180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922200.
00010000:00000010:0.1:1787760749.560310:0:282442:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19500.
00010000:00000010:0.1:1787760749.560316:0:282442:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104ce00.
00010000:00000010:1.1:1787760749.564172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f500.
00010000:00000010:1.1:1787760749.564184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ca00.
00010000:00000010:1.1:1787760749.568182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5800.
00010000:00000010:1.1:1787760749.568198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f800.
00000080:00000010:1.1:1787760749.736504:0:303458:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303fc0.
00010000:00000010:1.1:1787760749.736510:0:303458:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19c00.
00010000:00000010:1.1:1787760749.736514:0:303458:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104d600.
00010000:00000010:0.1:1787760749.743932:0:303314:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4fc00.
00000080:00000010:0.1:1787760749.796197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cad00.
00010000:00000010:1.1:1787760749.832649:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f500.
00010000:00000010:1.1:1787760749.832665:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ca00.
00000080:00000010:1.1:1787760749.855840:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc6e0.
00000080:00000010:1.1:1787760749.869594:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc6c0.
00000080:00000010:0.1:1787760749.904199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cab60.
00010000:00000010:0.1:1787760749.928189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5600.
00010000:00000010:0.1:1787760749.928211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e200.
00000080:00000010:0.1:1787760749.938510:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303da0.
00000080:00000010:0.1:1787760749.951275:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303480.
00000080:00000010:3.1:1787760750.015303:0:257571:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d200.
00010000:00000010:1.1:1787760750.035202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923c00.
00000080:00000010:1.1:1787760750.039415:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca460.
00010000:00000010:0.1:1787760750.090987:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e200.
00010000:00000010:1.1:1787760750.207970:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5200.
00010000:00000010:1.1:1787760750.207977:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4fc00.
00010000:00000010:0.1:1787760750.221178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f500.
00010000:00000010:0.1:1787760750.226148:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5100.
00010000:00000010:0.1:1787760750.226166:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d000.
00010000:00000010:0.1:1787760750.604186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d600.
00010000:00000010:0.1:1787760750.919352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e000.
00000080:00000010:0.1:1787760750.919370:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303fe0.
00000080:00000010:1.1:1787760750.930231:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303940.
00010000:00000010:0.1:1787760750.930286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e800.
00010000:00000010:1.1:1787760750.941032:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c200.
00000080:00000010:1.1:1787760750.944299:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc940.
00010000:00000010:1.1:1787760750.944321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846da00.
00010000:00000010:1.1:1787760750.944324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104d600.
00010000:00000010:1.1:1787760750.955180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f100.
00010000:00000010:1.1:1787760750.955199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d400.
00000080:00000010:1.1:1787760750.973446:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dce40.
00000080:00000010:1.1:1787760750.973463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d380.
00000080:00000010:0.1:1787760751.109573:0:303502:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d7c0.
00010000:00000010:0.1:1787760751.109592:0:303502:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d400.
00010000:00000010:0.1:1787760751.109599:0:303502:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e800.
00010000:00000010:0.1:1787760751.109606:0:303502:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5200.
00010000:00000010:0.1:1787760751.109611:0:303502:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c400.
00000080:00000010:0.1:1787760751.121277:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811dba0.
00010000:00000010:1.1:1787760751.195186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c600.
00010000:00000010:1.1:1787760751.195200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846cc00.
00010000:00000010:1.1:1787760751.195211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d800.
00010000:00000010:1.1:1787760751.195217:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5100.
00010000:00000010:1.1:1787760751.195222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e200.
00010000:00000010:1.1:1787760751.195227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922c00.
00000080:00000010:1.1:1787760751.202468:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc60.
00000080:00000010:1.1:1787760751.206540:0:303506:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1ae0840.
00010000:00000010:1.1:1787760754.627279:0:303787:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f100.
00010000:00000010:1.1:1787760754.627292:0:303787:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c200.
00010000:00000010:1.1:1787760757.996379:0:304092:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104d400.
00000080:00000010:1.1:1787760757.996419:0:304092:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5620.
00010000:00000010:1.1:1787760757.996429:0:304092:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922a00.
00010000:00000010:3.1:1787760758.226557:0:304097:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923800.
00000080:00000010:3.1:1787760758.226571:0:304097:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca2c0.
00010000:00000010:3.1:1787760758.252251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923800.
00010000:00000010:0.1:1787760758.266432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c200.
00000080:00000010:0.1:1787760758.266450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5240.
00000080:00000010:0.1:1787760758.405079:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303320.
00000080:00000010:0.1:1787760758.405112:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1ae0840.
00000080:00000010:0.1:1787760758.425301:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303bc0.
00010000:00000010:3.1:1787760758.479560:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5e00.
00010000:00000010:3.1:1787760758.479577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4fe00.
00000080:00000010:3.1:1787760758.495350:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303320.
00000080:00000010:3.1:1787760758.495369:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1ae0840.
00010000:00000010:0.1:1787760758.564870:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f900.
00010000:00000010:0.1:1787760758.564884:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104ca00.
00000080:00000010:0.1:1787760758.625172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5fe0.
00010000:00000010:0.1:1787760758.643656:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5c00.
00010000:00000010:0.1:1787760758.643668:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f600.
00000080:00000010:0.1:1787760758.668170:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303bc0.
00000080:00000010:0.1:1787760758.668183:0:304093:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1ae0840.
00000080:00000010:0.1:1787760758.673356:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303060.
00010000:00000010:1.1:1787760758.717888:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5c00.
00010000:00000010:1.1:1787760758.717893:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f600.
00000080:00000010:3.1:1787760758.721360:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303bc0.
00000080:00000010:3.1:1787760758.721381:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1ae0840.
00000080:00000010:3.1:1787760758.744432:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca080.
00010000:00000010:0.1:1787760758.776473:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f400.
00010000:00000010:0.1:1787760758.776488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922200.
00000080:00000010:0.1:1787760758.797744:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cade0.
00000080:00000010:0.1:1787760758.797764:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1016b40.
00010000:00000010:3.1:1787760758.827353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923800.
00000080:00000010:3.1:1787760758.827369:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca080.
00010000:00000010:0.1:1787760758.831552:0:304099:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104d000.
00000080:00000010:0.1:1787760758.831558:0:304099:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cace0.
00010000:00000010:0.1:1787760758.847547:0:304099:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e200.
00000080:00000010:0.1:1787760758.847555:0:304099:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303500.
00010000:00000010:0.1:1787760758.861499:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f800.
00000080:00000010:0.1:1787760758.861509:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303de0.
00010000:00000010:1.1:1787760758.876193:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5c00.
00010000:00000010:1.1:1787760758.876199:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c200.
00000080:00000010:1.1:1787760758.885328:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5e20.
00000080:00000010:1.1:1787760758.889317:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751d1ae0840.
00010000:00000010:3.1:1787760758.921086:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923c00.
00010000:00000010:1.1:1787760758.947426:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846dc00.
00000080:00000010:1.1:1787760758.947436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc320.
00010000:00000010:1.1:1787760758.958571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c800.
00000080:00000010:1.1:1787760758.958582:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcce0.
00010000:00000010:1.1:1787760759.017181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c800.
00000080:00000010:1.1:1787760759.017197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df55c0.
00010000:00000010:3.1:1787760759.045459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c800.
00010000:00000010:0.1:1787760759.074184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923c00.
00010000:00000010:0.1:1787760759.074192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419800.
00010000:00000010:0.1:1787760759.074196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104d400.
00000080:00000010:0.1:1787760759.082348:0:0:0:(super25.c:58:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9751e1071080.
00000080:00000010:0.1:1787760759.082362:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5700.
00010000:00000010:1.1:1787760759.100829:0:304104:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c800.
00000080:00000010:1.1:1787760759.100841:0:304104:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5220.
00010000:00000010:1.1:1787760759.106194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846dc00.
00000080:00000010:1.1:1787760759.106206:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc080.
00000080:00000010:1.1:1787760759.127402:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5e20.
00000080:00000010:1.1:1787760759.133305:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcbe0.
00010000:00000010:1.1:1787760759.174243:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5e00.
00010000:00000010:1.1:1787760759.174255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f800.
00000080:00000010:1.1:1787760759.197168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcbe0.
00000080:00000010:1.1:1787760759.197179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303be0.
00010000:00000010:0.1:1787760759.222332:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5800.
00010000:00000010:0.1:1787760759.222344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e200.
00000080:00000010:1.1:1787760759.226426:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303500.
00000080:00000010:1.1:1787760759.226441:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303de0.
00010000:00000010:1.1:1787760759.259340:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5800.
00010000:00000010:1.1:1787760759.259352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c600.
00000080:00000010:1.1:1787760759.264375:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc40.
00000080:00000010:1.1:1787760759.264391:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5360.
00010000:00000010:1.1:1787760759.313202:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f400.
00010000:00000010:1.1:1787760759.313213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c200.
00000080:00000010:1.1:1787760759.313219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcaa0.
00000080:00000010:1.1:1787760759.313224:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc40.
00010000:00000010:1.1:1787760759.330325:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f100.
00010000:00000010:1.1:1787760759.330335:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c400.
00000080:00000010:1.1:1787760759.340641:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc40.
00000080:00000010:1.1:1787760759.340660:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dccc0.
00000080:00000010:0.1:1787760759.397662:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5c80.
00000080:00000010:0.1:1787760759.397679:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dccc0.
00010000:00000010:0.1:1787760759.423563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5d00.
00010000:00000010:0.1:1787760759.423571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4fc00.
00000080:00000010:0.1:1787760759.436083:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303840.
00000080:00000010:0.1:1787760759.436094:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303800.
00010000:00000010:1.1:1787760759.491187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5400.
00010000:00000010:1.1:1787760759.491198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f400.
00000080:00000010:1.1:1787760759.491204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc40.
00000080:00000010:1.1:1787760759.491213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303600.
00010000:00000010:1.1:1787760759.540302:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419600.
00010000:00000010:1.1:1787760759.540315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ea00.
00000080:00000010:1.1:1787760759.551438:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc40.
00000080:00000010:1.1:1787760759.551454:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303800.
00010000:00000010:0.1:1787760759.623242:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5100.
00010000:00000010:0.1:1787760759.623257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c600.
00000080:00000010:1.1:1787760759.649599:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303840.
00000080:00000010:1.1:1787760759.649612:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc40.
00010000:00000010:0.1:1787760759.733193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f500.
00010000:00000010:0.1:1787760759.733211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ca00.
00000080:00000010:0.1:1787760759.744310:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303720.
00000080:00000010:0.1:1787760759.744325:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc40.
00010000:00000010:1.1:1787760759.804201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f400.
00010000:00000010:1.1:1787760759.804217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d400.
00000080:00000010:1.1:1787760759.809333:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc220.
00000080:00000010:1.1:1787760759.809348:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcec0.
00010000:00000010:1.1:1787760759.870188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19e00.
00010000:00000010:1.1:1787760759.870202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d600.
00000080:00000010:1.1:1787760759.890228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc220.
00000080:00000010:1.1:1787760759.890244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcec0.
00010000:00000010:0.1:1787760759.958168:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19d00.
00010000:00000010:0.1:1787760759.958183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d000.
00000080:00000010:1.1:1787760759.977356:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303720.
00000080:00000010:1.1:1787760759.977376:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcec0.
00010000:00000010:1.1:1787760760.029395:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19800.
00010000:00000010:1.1:1787760760.029403:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ca00.
00000080:00000010:1.1:1787760760.033294:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc00.
00000080:00000010:1.1:1787760760.033308:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcec0.
00010000:00000010:1.1:1787760760.148126:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419b00.
00010000:00000010:1.1:1787760760.148177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4fa00.
00000080:00000010:1.1:1787760760.150174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303460.
00000080:00000010:1.1:1787760760.150190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303f40.
00010000:00000010:1.1:1787760760.202355:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080800.
00010000:00000010:1.1:1787760760.202365:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2800.
00000080:00000010:1.1:1787760760.207383:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc980.
00000080:00000010:1.1:1787760760.207404:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc8c0.
00010000:00000010:0.1:1787760760.318525:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5800.
00010000:00000010:0.1:1787760760.318532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104d400.
00000080:00000010:0.1:1787760760.322326:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303820.
00000080:00000010:0.1:1787760760.322346:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5240.
00010000:00000010:1.1:1787760760.359563:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19f00.
00010000:00000010:1.1:1787760760.359577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f200.
00000080:00000010:1.1:1787760760.377920:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc8c0.
00000080:00000010:1.1:1787760760.377944:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303820.
00010000:00000010:0.1:1787760760.404532:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19100.
00010000:00000010:0.1:1787760760.404538:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3a00.
00000080:00000010:0.1:1787760760.418318:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303260.
00000080:00000010:0.1:1787760760.418327:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcd20.
00010000:00000010:0.1:1787760760.459629:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19600.
00010000:00000010:0.1:1787760760.459646:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3800.
00000080:00000010:0.1:1787760760.474570:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303260.
00000080:00000010:0.1:1787760760.474582:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc8c0.
00010000:00000010:0.1:1787760760.516843:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5400.
00010000:00000010:0.1:1787760760.516849:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f400.
00000080:00000010:0.1:1787760760.520374:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303e60.
00000080:00000010:0.1:1787760760.520392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303260.
00010000:00000010:0.1:1787760760.563582:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19b00.
00010000:00000010:0.1:1787760760.563589:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3000.
00000080:00000010:0.1:1787760760.570186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303260.
00000080:00000010:0.1:1787760760.570204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc260.
00010000:00000010:1.1:1787760760.605589:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9e00.
00010000:00000010:1.1:1787760760.605603:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f800.
00000080:00000010:1.1:1787760760.626336:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303f80.
00000080:00000010:1.1:1787760760.626348:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303260.
00010000:00000010:0.1:1787760760.658359:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19b00.
00010000:00000010:0.1:1787760760.658371:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2400.
00000080:00000010:0.1:1787760760.673102:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303fc0.
00000080:00000010:0.1:1787760760.673122:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc4e0.
00010000:00000010:1.1:1787760760.815180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419500.
00010000:00000010:1.1:1787760760.815194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104ce00.
00000080:00000010:1.1:1787760760.824310:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc420.
00000080:00000010:1.1:1787760760.824327:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5ce0.
00010000:00000010:1.1:1787760760.893230:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19100.
00010000:00000010:1.1:1787760760.893237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2c00.
00000080:00000010:1.1:1787760760.902182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc780.
00000080:00000010:1.1:1787760760.902200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc420.
00010000:00000010:0.1:1787760760.980183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19c00.
00010000:00000010:0.1:1787760760.980198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3400.
00000080:00000010:0.1:1787760760.986296:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303fc0.
00000080:00000010:0.1:1787760760.986315:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc420.
00010000:00000010:1.1:1787760761.041442:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9500.
00010000:00000010:1.1:1787760761.041449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ee00.
00000080:00000010:0.1:1787760761.063701:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc780.
00000080:00000010:0.1:1787760761.063721:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303fc0.
00010000:00000010:1.1:1787760761.108394:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9300.
00010000:00000010:1.1:1787760761.108411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e600.
00000080:00000010:3.1:1787760761.120333:0:257571:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc400.
00000080:00000010:3.1:1787760761.120342:0:257571:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303980.
00010000:00000010:0.1:1787760761.212224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9a00.
00010000:00000010:0.1:1787760761.212237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3c00.
00000080:00000010:0.1:1787760761.255253:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303fc0.
00000080:00000010:0.1:1787760761.255274:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc80.
00010000:00000010:1.1:1787760761.310379:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19b00.
00010000:00000010:1.1:1787760761.310393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2600.
00000080:00000010:1.1:1787760761.316242:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc040.
00000080:00000010:1.1:1787760761.316256:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc3a0.
00010000:00000010:0.1:1787760761.379301:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9a00.
00010000:00000010:0.1:1787760761.379307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104ce00.
00000080:00000010:0.1:1787760761.386305:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303fc0.
00000080:00000010:0.1:1787760761.386315:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5f20.
00010000:00000010:3.1:1787760761.431291:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9f00.
00010000:00000010:3.1:1787760761.431321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e600.
00000080:00000010:0.1:1787760761.447187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca720.
00000080:00000010:0.1:1787760761.447212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303fc0.
00010000:00000010:1.1:1787760761.511258:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9000.
00010000:00000010:1.1:1787760761.511269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f800.
00000080:00000010:1.1:1787760761.521366:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc3a0.
00000080:00000010:1.1:1787760761.521385:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303fc0.
00010000:00000010:0.1:1787760761.575241:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6b00.
00010000:00000010:0.1:1787760761.575253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2a00.
00000080:00000010:0.1:1787760761.586364:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93039e0.
00000080:00000010:0.1:1787760761.586381:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc3a0.
00010000:00000010:1.1:1787760761.641329:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9d00.
00010000:00000010:1.1:1787760761.641337:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f800.
00000080:00000010:1.1:1787760761.646331:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcd80.
00000080:00000010:1.1:1787760761.646351:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93039e0.
00010000:00000010:0.1:1787760761.691284:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6e00.
00010000:00000010:0.1:1787760761.691294:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3c00.
00000080:00000010:0.1:1787760761.696428:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303340.
00000080:00000010:0.1:1787760761.696441:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc900.
00010000:00000010:1.1:1787760761.768180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9b00.
00010000:00000010:1.1:1787760761.768195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f800.
00000080:00000010:1.1:1787760761.788324:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcd80.
00000080:00000010:1.1:1787760761.788337:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303340.
00010000:00000010:3.1:1787760761.843062:0:257570:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080300.
00010000:00000010:3.1:1787760761.843073:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2400.
00000080:00000010:3.1:1787760761.846190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0caa80.
00000080:00000010:3.1:1787760761.846211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcd80.
00010000:00000010:1.1:1787760761.925174:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6200.
00010000:00000010:1.1:1787760761.925182:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3000.
00000080:00000010:1.1:1787760761.942198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcf80.
00000080:00000010:1.1:1787760761.942212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc120.
00010000:00000010:0.1:1787760762.036434:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9100.
00010000:00000010:0.1:1787760762.036451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798bc00.
00000080:00000010:1.1:1787760762.036523:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303d40.
00000080:00000010:1.1:1787760762.036534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93030c0.
00010000:00000010:3.1:1787760762.191923:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419100.
00010000:00000010:3.1:1787760762.191940:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104ce00.
00000080:00000010:0.1:1787760762.202442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0caa80.
00000080:00000010:0.1:1787760762.202458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5700.
00010000:00000010:1.1:1787760762.258211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6a00.
00010000:00000010:1.1:1787760762.258226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3a00.
00000080:00000010:1.1:1787760762.265240:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcd80.
00000080:00000010:1.1:1787760762.265260:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcf80.
00010000:00000010:0.1:1787760762.325361:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6f00.
00010000:00000010:0.1:1787760762.325372:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2400.
00000080:00000010:0.1:1787760762.348187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303340.
00000080:00000010:0.1:1787760762.348204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcf80.
00010000:00000010:3.1:1787760762.412444:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9700.
00010000:00000010:3.1:1787760762.412462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b800.
00000080:00000010:0.1:1787760762.445194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca800.
00000080:00000010:0.1:1787760762.445206:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303fc0.
00010000:00000010:0.1:1787760762.503801:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419000.
00010000:00000010:0.1:1787760762.503813:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888200.
00000080:00000010:1.1:1787760762.588398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5c80.
00000080:00000010:1.1:1787760762.588418:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5060.
00010000:00000010:0.1:1787760762.660245:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9b00.
00010000:00000010:0.1:1787760762.660267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7889600.
00000080:00000010:0.1:1787760762.674046:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303340.
00000080:00000010:0.1:1787760762.674068:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5c40.
00010000:00000010:1.1:1787760762.720201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9d00.
00010000:00000010:1.1:1787760762.720212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798a000.
00000080:00000010:1.1:1787760762.728322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc80.
00000080:00000010:1.1:1787760762.728332:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303340.
00010000:00000010:1.1:1787760762.775184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080c00.
00010000:00000010:1.1:1787760762.775203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922600.
00000080:00000010:1.1:1787760762.786248:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc80.
00000080:00000010:1.1:1787760762.786259:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca080.
00010000:00000010:1.1:1787760762.831358:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080f00.
00010000:00000010:1.1:1787760762.831368:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923e00.
00000080:00000010:1.1:1787760762.854106:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc80.
00000080:00000010:1.1:1787760762.854123:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0caa40.
00010000:00000010:0.1:1787760762.878308:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9f00.
00010000:00000010:0.1:1787760762.878315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b000.
00000080:00000010:0.1:1787760762.887386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303820.
00000080:00000010:0.1:1787760762.887405:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303f80.
00010000:00000010:0.1:1787760762.939224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9300.
00010000:00000010:0.1:1787760762.939238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b000.
00000080:00000010:0.1:1787760762.939246:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303f80.
00000080:00000010:0.1:1787760762.939256:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303f40.
00010000:00000010:1.1:1787760762.984174:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9e00.
00010000:00000010:1.1:1787760762.984183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b200.
00000080:00000010:1.1:1787760762.984190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc80.
00000080:00000010:1.1:1787760762.984199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303460.
00010000:00000010:1.1:1787760763.020312:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6d00.
00010000:00000010:1.1:1787760763.020324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad07600.
00000080:00000010:1.1:1787760763.027311:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc80.
00000080:00000010:1.1:1787760763.027330:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc4e0.
00010000:00000010:1.1:1787760763.074900:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6500.
00010000:00000010:1.1:1787760763.074915:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06400.
00000080:00000010:1.1:1787760763.082902:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc80.
00000080:00000010:1.1:1787760763.082920:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc4e0.
00010000:00000010:1.1:1787760763.151377:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6c00.
00010000:00000010:1.1:1787760763.151391:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06400.
00000080:00000010:1.1:1787760763.167176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc80.
00000080:00000010:1.1:1787760763.167202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc4e0.
00010000:00000010:0.1:1787760763.217320:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6800.
00010000:00000010:0.1:1787760763.217327:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06000.
00000080:00000010:0.1:1787760763.224386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303f40.
00000080:00000010:0.1:1787760763.224405:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcec0.
00010000:00000010:3.1:1787760763.309179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6700.
00010000:00000010:3.1:1787760763.309194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad07c00.
00000080:00000010:3.1:1787760763.323536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0caac0.
00000080:00000010:3.1:1787760763.323555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc4e0.
00010000:00000010:1.1:1787760763.389180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9d00.
00010000:00000010:1.1:1787760763.389204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798a400.
00000080:00000010:1.1:1787760763.410275:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc80.
00000080:00000010:1.1:1787760763.410288:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303720.
00010000:00000010:1.1:1787760763.461490:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6800.
00010000:00000010:1.1:1787760763.461501:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad07800.
00000080:00000010:1.1:1787760763.469197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc700.
00000080:00000010:1.1:1787760763.469215:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc80.
00010000:00000010:0.1:1787760763.569176:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9700.
00010000:00000010:0.1:1787760763.569199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06200.
00000080:00000010:0.1:1787760763.577353:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303840.
00000080:00000010:0.1:1787760763.577376:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcc80.
00010000:00000010:1.1:1787760763.638549:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6600.
00010000:00000010:1.1:1787760763.638557:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06c00.
00000080:00000010:0.1:1787760763.645437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcb40.
00000080:00000010:0.1:1787760763.645456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc320.
00010000:00000010:1.1:1787760763.704285:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9d00.
00010000:00000010:1.1:1787760763.704292:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798a400.
00000080:00000010:1.1:1787760763.713334:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcf60.
00000080:00000010:1.1:1787760763.713348:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93039a0.
00010000:00000010:1.1:1787760763.798774:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6e00.
00010000:00000010:1.1:1787760763.798787:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06800.
00000080:00000010:1.1:1787760763.801305:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc2e0.
00000080:00000010:1.1:1787760763.801322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcf60.
00010000:00000010:0.1:1787760763.850667:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6b00.
00010000:00000010:0.1:1787760763.850680:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7889000.
00000080:00000010:0.1:1787760763.861259:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303800.
00000080:00000010:0.1:1787760763.861271:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5060.
00000080:00000010:1.1:1787760763.926556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5c80.
00000080:00000010:1.1:1787760763.926569:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc2e0.
00010000:00000010:1.1:1787760763.966435:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975202160400.
00010000:00000010:1.1:1787760763.966444:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06000.
00000080:00000010:1.1:1787760763.987171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc2e0.
00000080:00000010:1.1:1787760763.987182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc940.
00010000:00000010:0.1:1787760764.036263:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975202160900.
00010000:00000010:0.1:1787760764.036279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f57800.
00000080:00000010:0.1:1787760764.039356:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303800.
00000080:00000010:0.1:1787760764.039370:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc1c0.
00010000:00000010:1.1:1787760764.104763:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1700.
00010000:00000010:1.1:1787760764.104770:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798a000.
00000080:00000010:1.1:1787760764.108799:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc940.
00000080:00000010:1.1:1787760764.108821:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303800.
00010000:00000010:0.1:1787760764.159486:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975202160100.
00010000:00000010:0.1:1787760764.159503:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56400.
00000080:00000010:0.1:1787760764.176192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303de0.
00000080:00000010:0.1:1787760764.176211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcca0.
00010000:00000010:1.1:1787760764.234385:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9d00.
00010000:00000010:1.1:1787760764.234397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f57000.
00000080:00000010:1.1:1787760764.242497:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dca80.
00000080:00000010:1.1:1787760764.242514:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc9a0.
00000080:00000010:1.1:1787760764.370218:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5860.
00000080:00000010:1.1:1787760764.370237:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5340.
00010000:00000010:1.1:1787760764.442570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0600.
00010000:00000010:1.1:1787760764.442585:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56000.
00000080:00000010:1.1:1787760764.452408:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc4a0.
00000080:00000010:1.1:1787760764.452427:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dca80.
00010000:00000010:3.1:1787760764.546746:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080500.
00010000:00000010:3.1:1787760764.546756:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888800.
00000080:00000010:3.1:1787760764.564202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0caac0.
00000080:00000010:3.1:1787760764.564619:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5b20.
00010000:00000010:1.1:1787760764.646184:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1500.
00010000:00000010:1.1:1787760764.646192:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b800.
00000080:00000010:1.1:1787760764.665240:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dca80.
00000080:00000010:1.1:1787760764.665271:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303de0.
00010000:00000010:0.1:1787760764.752538:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0100.
00010000:00000010:0.1:1787760764.752554:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56c00.
00000080:00000010:0.1:1787760764.787917:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303500.
00000080:00000010:0.1:1787760764.787938:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc0a0.
00000080:00000010:0.1:1787760764.900188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5180.
00000080:00000010:0.1:1787760764.900208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df52a0.
00010000:00000010:1.1:1787760764.974199:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1e00.
00010000:00000010:1.1:1787760764.974214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888a00.
00000080:00000010:1.1:1787760765.005192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dca40.
00000080:00000010:1.1:1787760765.005204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5460.
00000080:00000010:1.1:1787760765.084165:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5ce0.
00000080:00000010:1.1:1787760765.084179:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303060.
00010000:00000010:1.1:1787760765.142531:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0c00.
00010000:00000010:1.1:1787760765.142541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f57200.
00000080:00000010:1.1:1787760765.150340:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dce80.
00000080:00000010:1.1:1787760765.150357:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc280.
00010000:00000010:0.1:1787760765.223520:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0b00.
00010000:00000010:0.1:1787760765.223529:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f57200.
00000080:00000010:0.1:1787760765.231210:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303bc0.
00000080:00000010:0.1:1787760765.231227:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc1e0.
00010000:00000010:1.1:1787760765.314192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1100.
00010000:00000010:1.1:1787760765.314204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b800.
00000080:00000010:1.1:1787760765.314211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc280.
00000080:00000010:1.1:1787760765.314220:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303bc0.
00000080:00000010:1.1:1787760765.390750:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303d20.
00000080:00000010:1.1:1787760765.390769:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dce80.
00010000:00000010:0.1:1787760765.391297:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0d00.
00010000:00000010:0.1:1787760765.391305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56a00.
00010000:00000010:1.1:1787760765.459000:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0e00.
00010000:00000010:1.1:1787760765.459025:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56a00.
00000080:00000010:0.1:1787760765.472181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc720.
00000080:00000010:0.1:1787760765.472190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dce80.
00010000:00000010:1.1:1787760765.562347:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1800.
00010000:00000010:1.1:1787760765.562361:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b800.
00000080:00000010:1.1:1787760765.576161:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcd40.
00000080:00000010:1.1:1787760765.576172:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93030e0.
00010000:00000010:0.1:1787760765.650319:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0300.
00010000:00000010:0.1:1787760765.650335:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f57200.
00000080:00000010:1.1:1787760765.666271:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303640.
00000080:00000010:1.1:1787760765.666290:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcd40.
00010000:00000010:1.1:1787760765.751334:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0700.
00010000:00000010:1.1:1787760765.751348:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56400.
00000080:00000010:1.1:1787760765.769512:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcac0.
00000080:00000010:1.1:1787760765.769533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcd40.
00010000:00000010:1.1:1787760765.884336:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0500.
00010000:00000010:1.1:1787760765.884356:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56600.
00000080:00000010:1.1:1787760765.911361:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcac0.
00000080:00000010:1.1:1787760765.911386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcd40.
00010000:00000010:0.1:1787760766.044195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080d00.
00010000:00000010:0.1:1787760766.044338:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923600.
00000080:00000010:0.1:1787760766.087186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303d60.
00000080:00000010:0.1:1787760766.092227:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0caac0.
00010000:00000010:1.1:1787760766.205027:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419b00.
00010000:00000010:1.1:1787760766.205043:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922c00.
00000080:00000010:1.1:1787760766.227241:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcd40.
00000080:00000010:1.1:1787760766.227283:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca560.
00010000:00000010:0.1:1787760766.332255:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1100.
00010000:00000010:0.1:1787760766.332266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441200.
00000080:00000010:0.1:1787760766.353303:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303380.
00000080:00000010:0.1:1787760766.353350:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303d60.
00010000:00000010:0.1:1787760766.561481:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1900.
00010000:00000010:0.1:1787760766.561503:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888800.
00000080:00000010:0.1:1787760766.575692:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303d60.
00000080:00000010:0.1:1787760766.575717:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5e20.
00010000:00000010:0.1:1787760766.677202:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0a00.
00010000:00000010:0.1:1787760766.677221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f57200.
00000080:00000010:0.1:1787760766.703178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303d60.
00000080:00000010:0.1:1787760766.703193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcd40.
00010000:00000010:0.1:1787760766.811212:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1e00.
00010000:00000010:0.1:1787760766.811234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441e00.
00000080:00000010:0.1:1787760766.841262:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303860.
00000080:00000010:0.1:1787760766.841276:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303d60.
00010000:00000010:1.1:1787760766.961873:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1a00.
00010000:00000010:1.1:1787760766.961895:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441c00.
00000080:00000010:1.1:1787760767.016198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc9c0.
00000080:00000010:1.1:1787760767.016229:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303d60.
00010000:00000010:1.1:1787760767.163190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0400.
00010000:00000010:1.1:1787760767.163210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440600.
00010000:00000010:0.1:1787760767.281183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1a00.
00010000:00000010:0.1:1787760767.281199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441800.
00000080:00000010:0.1:1787760767.310713:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303b20.
00000080:00000010:0.1:1787760767.310726:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303f60.
00010000:00000010:1.1:1787760767.409178:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101400.
00010000:00000010:1.1:1787760767.409186:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888e00.
00000080:00000010:1.1:1787760767.414311:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc180.
00000080:00000010:1.1:1787760767.414329:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df56e0.
00010000:00000010:0.1:1787760767.511931:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0a00.
00010000:00000010:0.1:1787760767.511946:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58600.
00000080:00000010:0.1:1787760767.521485:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303f60.
00000080:00000010:0.1:1787760767.521506:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc180.
00010000:00000010:1.1:1787760767.620347:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0300.
00010000:00000010:1.1:1787760767.620357:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441600.
00000080:00000010:1.1:1787760767.634250:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc6e0.
00000080:00000010:1.1:1787760767.634267:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303220.
00010000:00000010:1.1:1787760767.701407:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101600.
00010000:00000010:1.1:1787760767.701423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888e00.
00000080:00000010:1.1:1787760767.740080:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc6e0.
00000080:00000010:1.1:1787760767.740103:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df51a0.
00010000:00000010:3.1:1787760767.873179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101100.
00010000:00000010:3.1:1787760767.873194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888a00.
00000080:00000010:3.1:1787760767.896202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca680.
00000080:00000010:3.1:1787760767.900266:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5460.
00010000:00000010:1.1:1787760768.220989:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101a00.
00010000:00000010:1.1:1787760768.221004:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922e00.
00000080:00000010:0.1:1787760768.243229:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc6e0.
00000080:00000010:0.1:1787760768.243239:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca060.
00010000:00000010:0.1:1787760768.436870:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49e00.
00010000:00000010:0.1:1787760768.436884:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888a00.
00000080:00000010:0.1:1787760768.451331:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303560.
00000080:00000010:0.1:1787760768.451344:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5ce0.
00010000:00000010:1.1:1787760768.525186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101e00.
00010000:00000010:1.1:1787760768.525200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888e00.
00000080:00000010:1.1:1787760768.540343:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcee0.
00000080:00000010:1.1:1787760768.540359:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5880.
00010000:00000010:1.1:1787760768.636187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0b00.
00010000:00000010:1.1:1787760768.636202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58c00.
00000080:00000010:1.1:1787760768.662180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc3c0.
00000080:00000010:1.1:1787760768.662192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcee0.
00010000:00000010:0.1:1787760768.996727:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49d00.
00010000:00000010:0.1:1787760768.996749:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888e00.
00000080:00000010:0.1:1787760769.023176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93034c0.
00000080:00000010:0.1:1787760769.023203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5e40.
00000080:00000010:0.1:1787760769.192090:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5460.
00000080:00000010:0.1:1787760769.192119:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcb00.
00010000:00000010:0.1:1787760769.322335:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0a00.
00010000:00000010:0.1:1787760769.322356:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58000.
00000080:00000010:0.1:1787760769.344173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93034c0.
00000080:00000010:0.1:1787760769.344190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc960.
00010000:00000010:0.1:1787760769.495834:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90500.
00010000:00000010:0.1:1787760769.495843:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58200.
00000080:00000010:0.1:1787760769.505429:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93034c0.
00000080:00000010:0.1:1787760769.505447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc820.
00010000:00000010:0.1:1787760769.635333:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90700.
00010000:00000010:0.1:1787760769.635349:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58000.
00000080:00000010:0.1:1787760769.645176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93034c0.
00000080:00000010:0.1:1787760769.645193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc5a0.
00010000:00000010:0.1:1787760769.707360:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90200.
00010000:00000010:0.1:1787760769.707368:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58c00.
00000080:00000010:0.1:1787760769.719485:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93034c0.
00000080:00000010:0.1:1787760769.719498:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc7a0.
00010000:00000010:1.1:1787760769.786750:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101c00.
00010000:00000010:1.1:1787760769.786765:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440400.
00000080:00000010:1.1:1787760769.796859:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcbc0.
00000080:00000010:1.1:1787760769.796880:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93034c0.
00010000:00000010:0.1:1787760769.862606:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101d00.
00010000:00000010:0.1:1787760769.862612:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58600.
00000080:00000010:0.1:1787760769.871256:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303660.
00000080:00000010:0.1:1787760769.871277:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcdc0.
00010000:00000010:0.1:1787760770.043175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080100.
00010000:00000010:0.1:1787760770.043184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7889c00.
00000080:00000010:0.1:1787760770.050323:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303660.
00000080:00000010:0.1:1787760770.050334:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5760.
00010000:00000010:0.1:1787760770.108267:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90000.
00010000:00000010:0.1:1787760770.108274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440600.
00000080:00000010:0.1:1787760770.112173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303660.
00000080:00000010:0.1:1787760770.112192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93034a0.
00010000:00000010:0.1:1787760770.177264:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101c00.
00010000:00000010:0.1:1787760770.177280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440600.
00000080:00000010:0.1:1787760770.193610:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303660.
00000080:00000010:0.1:1787760770.193637:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93034a0.
00000080:00000010:0.1:1787760770.249321:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df56e0.
00000080:00000010:0.1:1787760770.249341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303920.
00010000:00000010:1.1:1787760770.345418:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101a00.
00010000:00000010:1.1:1787760770.345432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440400.
00000080:00000010:0.1:1787760770.361348:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcbc0.
00000080:00000010:0.1:1787760770.361368:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303920.
00010000:00000010:1.1:1787760770.434256:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101600.
00010000:00000010:1.1:1787760770.434283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8e00.
00000080:00000010:1.1:1787760770.452348:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcfe0.
00000080:00000010:1.1:1787760770.452360:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303920.
00010000:00000010:0.1:1787760770.548189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101600.
00010000:00000010:0.1:1787760770.548201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed9e00.
00000080:00000010:0.1:1787760770.580203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303ae0.
00000080:00000010:0.1:1787760770.580217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93038c0.
00010000:00000010:1.1:1787760770.707178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90900.
00010000:00000010:1.1:1787760770.707192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58600.
00000080:00000010:1.1:1787760770.718066:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc7e0.
00000080:00000010:1.1:1787760770.718087:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcfe0.
00010000:00000010:0.1:1787760770.800412:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080000.
00010000:00000010:0.1:1787760770.800428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923200.
00000080:00000010:0.1:1787760770.818179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93038c0.
00000080:00000010:0.1:1787760770.818194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0caa00.
00010000:00000010:1.1:1787760770.904276:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080a00.
00010000:00000010:1.1:1787760770.904295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923400.
00000080:00000010:3.1:1787760770.930228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc7e0.
00000080:00000010:3.1:1787760770.930254:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca5e0.
00010000:00000010:0.1:1787760771.051227:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90900.
00010000:00000010:0.1:1787760771.051237:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922400.
00000080:00000010:0.1:1787760771.057293:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93038c0.
00000080:00000010:0.1:1787760771.057310:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca5e0.
00010000:00000010:1.1:1787760771.146275:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101e00.
00010000:00000010:1.1:1787760771.146300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed9200.
00000080:00000010:0.1:1787760771.164601:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc5c0.
00000080:00000010:0.1:1787760771.164795:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93038c0.
00010000:00000010:0.1:1787760771.242407:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9300.
00010000:00000010:0.1:1787760771.242414:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8600.
00000080:00000010:0.1:1787760771.261601:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303480.
00000080:00000010:0.1:1787760771.261630:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93038c0.
00010000:00000010:1.1:1787760771.341302:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9d00.
00010000:00000010:1.1:1787760771.341314:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8000.
00010000:00000010:0.1:1787760771.492169:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080000.
00010000:00000010:0.1:1787760771.492185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923000.
00000080:00000010:0.1:1787760771.502824:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93038c0.
00000080:00000010:0.1:1787760771.502838:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca8a0.
00010000:00000010:1.1:1787760771.575377:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9e00.
00010000:00000010:1.1:1787760771.575388:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8400.
00000080:00000010:1.1:1787760771.580278:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc500.
00000080:00000010:1.1:1787760771.580290:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93037c0.
00010000:00000010:0.1:1787760771.661366:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90000.
00010000:00000010:0.1:1787760771.661373:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58000.
00000080:00000010:0.1:1787760771.674479:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93038c0.
00000080:00000010:0.1:1787760771.674499:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc500.
00010000:00000010:1.1:1787760771.765993:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9c00.
00010000:00000010:1.1:1787760771.766007:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8200.
00000080:00000010:1.1:1787760771.803065:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc9e0.
00000080:00000010:1.1:1787760771.817561:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93038c0.
00010000:00000010:0.1:1787760771.898612:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9f00.
00010000:00000010:0.1:1787760771.898627:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed9800.
00000080:00000010:0.1:1787760771.899833:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303aa0.
00000080:00000010:0.1:1787760771.899844:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303e20.
00010000:00000010:1.1:1787760771.956391:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9b00.
00010000:00000010:1.1:1787760771.956398:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed9800.
00000080:00000010:1.1:1787760771.966250:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc9e0.
00000080:00000010:1.1:1787760771.966262:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303e20.
00010000:00000010:0.1:1787760772.043470:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90700.
00010000:00000010:0.1:1787760772.043480:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c0031600.
00000080:00000010:0.1:1787760772.049244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303aa0.
00000080:00000010:0.1:1787760772.049254:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc9e0.
00010000:00000010:1.1:1787760772.180251:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9900.
00010000:00000010:1.1:1787760772.180265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8a00.
00000080:00000010:1.1:1787760772.196170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcf20.
00000080:00000010:1.1:1787760772.196180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303aa0.
00010000:00000010:0.1:1787760772.296181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9900.
00010000:00000010:0.1:1787760772.296192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888800.
00000080:00000010:1.1:1787760772.307413:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303740.
00000080:00000010:1.1:1787760772.307425:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5700.
00010000:00000010:0.1:1787760772.385229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90000.
00010000:00000010:0.1:1787760772.385244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c0031200.
00000080:00000010:0.1:1787760772.401379:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303ca0.
00000080:00000010:0.1:1787760772.401398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc9e0.
00010000:00000010:0.1:1787760772.491436:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90000.
00010000:00000010:0.1:1787760772.491445:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888a00.
00000080:00000010:0.1:1787760772.505172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303ca0.
00000080:00000010:0.1:1787760772.505183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5860.
00010000:00000010:1.1:1787760772.570177:0:13:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9b00.
00010000:00000010:1.1:1787760772.570190:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8200.
00000080:00000010:1.1:1787760772.575192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc200.
00000080:00000010:1.1:1787760772.575210:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303940.
00010000:00000010:1.1:1787760772.630172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742e00.
00010000:00000010:1.1:1787760772.630179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c0031e00.
00000080:00000010:1.1:1787760772.639267:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc200.
00000080:00000010:1.1:1787760772.639282:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcb80.
00010000:00000010:1.1:1787760772.717556:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9c00.
00010000:00000010:1.1:1787760772.717570:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8400.
00000080:00000010:1.1:1787760772.753254:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcb80.
00000080:00000010:1.1:1787760772.753271:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca8e0.
00010000:00000010:0.1:1787760772.829173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742400.
00010000:00000010:0.1:1787760772.829188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c0031600.
00000080:00000010:0.1:1787760772.832347:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303ca0.
00000080:00000010:0.1:1787760772.832358:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcb80.
00010000:00000010:1.1:1787760772.903344:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742100.
00010000:00000010:1.1:1787760772.903352:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed9200.
00000080:00000010:1.1:1787760772.917185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc7e0.
00000080:00000010:1.1:1787760772.917203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93034e0.
00010000:00000010:1.1:1787760773.025178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742500.
00010000:00000010:1.1:1787760773.025194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f3e00.
00000080:00000010:0.1:1787760773.218567:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5ce0.
00000080:00000010:0.1:1787760773.218580:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc7a0.
00010000:00000010:0.1:1787760773.347514:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49100.
00010000:00000010:0.1:1787760773.347538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888800.
00000080:00000010:0.1:1787760773.362693:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303b40.
00000080:00000010:0.1:1787760773.362712:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5e40.
00010000:00000010:1.1:1787760773.490180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742900.
00010000:00000010:1.1:1787760773.490449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed9200.
00000080:00000010:1.1:1787760773.519193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc5a0.
00000080:00000010:1.1:1787760773.519213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303b40.
00010000:00000010:0.1:1787760773.626660:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742700.
00010000:00000010:0.1:1787760773.626676:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f3c00.
00000080:00000010:0.1:1787760773.652576:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93036c0.
00000080:00000010:0.1:1787760773.652604:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc5a0.
00010000:00000010:0.1:1787760773.798378:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080f00.
00010000:00000010:0.1:1787760773.798394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922400.
00000080:00000010:0.1:1787760773.806149:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93036c0.
00000080:00000010:0.1:1787760773.806161:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca880.
00010000:00000010:0.1:1787760773.880265:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742b00.
00010000:00000010:0.1:1787760773.880280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f2000.
00000080:00000010:0.1:1787760773.894182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93036c0.
00000080:00000010:0.1:1787760773.894194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc960.
00010000:00000010:1.1:1787760773.959173:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742600.
00010000:00000010:1.1:1787760773.959181:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8200.
00000080:00000010:1.1:1787760773.976210:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcb00.
00000080:00000010:1.1:1787760773.976223:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93036c0.
00010000:00000010:0.1:1787760774.062646:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742300.
00010000:00000010:0.1:1787760774.062661:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f2200.
00000080:00000010:0.1:1787760774.077422:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303300.
00000080:00000010:0.1:1787760774.077433:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc2c0.
00010000:00000010:1.1:1787760774.178773:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742800.
00010000:00000010:1.1:1787760774.178787:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f2c00.
00000080:00000010:1.1:1787760774.179306:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcb00.
00000080:00000010:1.1:1787760774.179319:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303b40.
00000080:00000010:0.1:1787760774.259215:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df55c0.
00000080:00000010:0.1:1787760774.259234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93034e0.
00010000:00000010:1.1:1787760774.322241:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5500.
00010000:00000010:1.1:1787760774.322252:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6c800.
00000080:00000010:1.1:1787760774.330411:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcb00.
00000080:00000010:1.1:1787760774.330421:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303740.
00010000:00000010:0.1:1787760774.397167:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742c00.
00010000:00000010:0.1:1787760774.397176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f2a00.
00000080:00000010:1.1:1787760774.397227:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93034e0.
00000080:00000010:1.1:1787760774.397239:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc180.
00010000:00000010:0.1:1787760774.481818:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742200.
00010000:00000010:0.1:1787760774.481833:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f2000.
00000080:00000010:0.1:1787760774.491472:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303aa0.
00000080:00000010:0.1:1787760774.491493:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc180.
00010000:00000010:1.1:1787760774.590193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742c00.
00010000:00000010:1.1:1787760774.590210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f3c00.
00000080:00000010:1.1:1787760774.610824:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcd40.
00000080:00000010:1.1:1787760774.610840:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc9c0.
00010000:00000010:0.1:1787760774.711195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742300.
00010000:00000010:0.1:1787760774.711212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f3e00.
00000080:00000010:0.1:1787760774.724655:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303aa0.
00000080:00000010:0.1:1787760774.724680:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc9c0.
00010000:00000010:1.1:1787760774.794297:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5300.
00010000:00000010:1.1:1787760774.794313:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d200.
00000080:00000010:1.1:1787760774.810297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dce80.
00000080:00000010:1.1:1787760774.810309:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303aa0.
00010000:00000010:0.1:1787760774.893402:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49500.
00010000:00000010:0.1:1787760774.893409:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468e00.
00010000:00000010:0.1:1787760775.002228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742b00.
00010000:00000010:0.1:1787760775.002242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f3e00.
00000080:00000010:0.1:1787760775.026171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303c80.
00000080:00000010:0.1:1787760775.026190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dce80.
00010000:00000010:3.1:1787760775.352334:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49500.
00010000:00000010:3.1:1787760775.352348:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923000.
00000080:00000010:3.1:1787760775.377256:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cad00.
00000080:00000010:3.1:1787760775.377277:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca440.
00000080:00000010:0.1:1787760775.566198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5da0.
00000080:00000010:0.1:1787760775.566217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc0a0.
00010000:00000010:0.1:1787760775.664794:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5e00.
00010000:00000010:0.1:1787760775.664808:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6dc00.
00000080:00000010:0.1:1787760775.683185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303220.
00000080:00000010:0.1:1787760775.683202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93035e0.
00010000:00000010:0.1:1787760775.851287:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742b00.
00010000:00000010:0.1:1787760775.851296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49a00.
00000080:00000010:0.1:1787760775.876282:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93035e0.
00000080:00000010:0.1:1787760775.876300:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc1c0.
00010000:00000010:1.1:1787760775.974411:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742b00.
00010000:00000010:1.1:1787760775.974421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48400.
00000080:00000010:1.1:1787760775.986202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcae0.
00000080:00000010:1.1:1787760775.986214:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcde0.
00010000:00000010:0.1:1787760776.168217:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5400.
00010000:00000010:0.1:1787760776.168232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d600.
00000080:00000010:0.1:1787760776.188220:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303d60.
00000080:00000010:0.1:1787760776.188239:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303860.
00010000:00000010:1.1:1787760776.321306:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4b500.
00010000:00000010:1.1:1787760776.321320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49000.
00000080:00000010:1.1:1787760776.349175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc320.
00000080:00000010:1.1:1787760776.349186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcae0.
00010000:00000010:0.1:1787760776.496356:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5400.
00010000:00000010:0.1:1787760776.496367:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468200.
00000080:00000010:0.1:1787760776.511281:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303d60.
00000080:00000010:0.1:1787760776.511291:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5500.
00010000:00000010:1.1:1787760776.584628:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4bd00.
00010000:00000010:1.1:1787760776.584636:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48c00.
00000080:00000010:1.1:1787760776.584642:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcb40.
00000080:00000010:1.1:1787760776.584649:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc320.
00010000:00000010:0.1:1787760776.670187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4bb00.
00010000:00000010:0.1:1787760776.670202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48c00.
00000080:00000010:0.1:1787760776.687186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303d60.
00000080:00000010:0.1:1787760776.687205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc320.
00010000:00000010:0.1:1787760776.796179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4b200.
00010000:00000010:0.1:1787760776.796192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49200.
00000080:00000010:0.1:1787760776.811187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303d60.
00000080:00000010:0.1:1787760776.811203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc4e0.
00010000:00000010:3.1:1787760776.961299:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49e00.
00010000:00000010:3.1:1787760776.961313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe469800.
00000080:00000010:0.1:1787760776.990835:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca240.
00000080:00000010:0.1:1787760776.990858:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5e20.
00010000:00000010:1.1:1787760777.140764:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4be00.
00010000:00000010:1.1:1787760777.140779:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49600.
00000080:00000010:1.1:1787760777.171689:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcf80.
00000080:00000010:1.1:1787760777.171710:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc3a0.
00010000:00000010:1.1:1787760777.337946:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5c00.
00010000:00000010:1.1:1787760777.337959:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d800.
00000080:00000010:1.1:1787760777.345907:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc3a0.
00000080:00000010:1.1:1787760777.345929:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303640.
00010000:00000010:0.1:1787760777.483465:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5800.
00010000:00000010:0.1:1787760777.483474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6da00.
00010000:00000010:1.1:1787760777.585347:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49500.
00010000:00000010:1.1:1787760777.585365:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468600.
00000080:00000010:1.1:1787760777.597265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc3a0.
00000080:00000010:1.1:1787760777.597277:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5e80.
00010000:00000010:1.1:1787760777.732305:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4b300.
00010000:00000010:1.1:1787760777.732320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49c00.
00000080:00000010:1.1:1787760777.756474:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc320.
00000080:00000010:1.1:1787760777.756490:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc3a0.
00010000:00000010:3.1:1787760777.841481:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4b100.
00010000:00000010:3.1:1787760777.841506:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48600.
00000080:00000010:3.1:1787760777.858506:0:257574:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca5a0.
00000080:00000010:3.1:1787760777.858517:0:257574:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc3a0.
00010000:00000010:0.1:1787760777.936183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5c00.
00010000:00000010:0.1:1787760777.936197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6da00.
00000080:00000010:0.1:1787760777.947247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303de0.
00000080:00000010:0.1:1787760777.947258:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303060.
00010000:00000010:0.1:1787760778.045016:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5b00.
00010000:00000010:0.1:1787760778.045030:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d800.
00010000:00000010:0.1:1787760778.247404:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5b00.
00010000:00000010:0.1:1787760778.247418:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923000.
00000080:00000010:0.1:1787760778.308185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93039a0.
00000080:00000010:0.1:1787760778.308208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cabe0.
00010000:00000010:1.1:1787760778.479707:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5b00.
00010000:00000010:1.1:1787760778.479723:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922e00.
00000080:00000010:1.1:1787760778.492180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc320.
00000080:00000010:1.1:1787760778.492193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca760.
00010000:00000010:1.1:1787760778.662257:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5300.
00010000:00000010:1.1:1787760778.662265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468200.
00000080:00000010:0.1:1787760778.666188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc320.
00000080:00000010:0.1:1787760778.666208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5040.
00000080:00000010:0.1:1787760778.808256:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5700.
00000080:00000010:0.1:1787760778.808269:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303720.
00010000:00000010:0.1:1787760778.970857:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4bf00.
00010000:00000010:0.1:1787760778.970872:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48600.
00000080:00000010:0.1:1787760779.020320:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303720.
00000080:00000010:0.1:1787760779.020332:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc0a0.
00010000:00000010:0.1:1787760779.221170:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4b700.
00010000:00000010:0.1:1787760779.221190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49200.
00000080:00000010:0.1:1787760779.257213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303720.
00000080:00000010:0.1:1787760779.257227:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc9c0.
00000080:00000010:2.1:1787760779.447732:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5760.
00010000:00000010:0.1:1787760779.544175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4bc00.
00010000:00000010:0.1:1787760779.544190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49a00.
00000080:00000010:0.1:1787760779.574168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303720.
00000080:00000010:1.1:1787760779.574757:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc180.
00010000:00000010:0.1:1787760779.656071:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4bc00.
00010000:00000010:0.1:1787760779.656085:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49200.
00000080:00000010:0.1:1787760779.665361:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303720.
00000080:00000010:0.1:1787760779.665373:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc180.
00010000:00000010:2.1:1787760779.782238:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080f00.
00010000:00000010:2.1:1787760779.782257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922c00.
00000080:00000010:3.1:1787760779.794680:0:257571:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5540.
00000080:00000010:3.1:1787760779.794700:0:257571:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cab80.
00010000:00000010:0.1:1787760779.942175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5b00.
00010000:00000010:0.1:1787760779.942205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d200.
00000080:00000010:0.1:1787760779.990535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303340.
00000080:00000010:0.1:1787760779.990558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303720.
00010000:00000010:2.1:1787760780.136178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d700.
00010000:00000010:2.1:1787760780.136195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d200.
00000080:00000010:0.1:1787760780.180187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5460.
00000080:00000010:0.1:1787760780.180210:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303720.
00010000:00000010:0.1:1787760780.350169:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d900.
00010000:00000010:0.1:1787760780.350176:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6ca00.
00000080:00000010:0.1:1787760780.370173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303fc0.
00000080:00000010:0.1:1787760780.370196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303720.
00010000:00000010:1.1:1787760780.552548:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5700.
00010000:00000010:1.1:1787760780.552564:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923000.
00000080:00000010:1.1:1787760780.602854:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc960.
00000080:00000010:1.1:1787760780.602873:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca760.
00010000:00000010:0.1:1787760780.766192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6dc00.
00010000:00000010:0.1:1787760780.766206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6ca00.
00000080:00000010:0.1:1787760780.790446:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303fc0.
00000080:00000010:0.1:1787760780.790461:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303720.
00010000:00000010:0.1:1787760780.974181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4bf00.
00010000:00000010:0.1:1787760780.974196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6ae800.
00000080:00000010:1.1:1787760780.998534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303720.
00000080:00000010:1.1:1787760780.998553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc7a0.
00010000:00000010:1.1:1787760781.145625:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4b200.
00010000:00000010:1.1:1787760781.145640:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6ae000.
00000080:00000010:0.1:1787760781.170645:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dcb80.
00000080:00000010:0.1:1787760781.170657:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc7a0.
00010000:00000010:1.1:1787760781.341632:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d500.
00010000:00000010:1.1:1787760781.341649:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d200.
00000080:00000010:1.1:1787760781.354646:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc9e0.
00000080:00000010:1.1:1787760781.354667:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93030c0.
00010000:00000010:1.1:1787760781.466289:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d500.
00010000:00000010:1.1:1787760781.466303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d200.
00000080:00000010:1.1:1787760781.493224:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc9e0.
00000080:00000010:1.1:1787760781.493237:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93030c0.
00010000:00000010:2.1:1787760781.605666:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d500.
00010000:00000010:2.1:1787760781.605685:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f200.
00000080:00000010:2.1:1787760781.615252:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df57a0.
00000080:00000010:2.1:1787760781.615264:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303060.
00010000:00000010:0.1:1787760781.811197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183300.
00010000:00000010:0.1:1787760781.811212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468800.
00000080:00000010:2.1:1787760781.820156:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303de0.
00000080:00000010:2.1:1787760781.820179:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5120.
00010000:00000010:2.1:1787760781.987176:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183e00.
00010000:00000010:2.1:1787760781.987185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468800.
00000080:00000010:2.1:1787760781.997147:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5a60.
00000080:00000010:2.1:1787760781.997178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5da0.
00010000:00000010:1.1:1787760782.128309:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5100.
00010000:00000010:1.1:1787760782.128324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e400.
00000080:00000010:1.1:1787760782.152177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc9e0.
00000080:00000010:1.1:1787760782.152191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303d20.
00010000:00000010:0.1:1787760782.285725:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5000.
00010000:00000010:0.1:1787760782.285746:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe469800.
00000080:00000010:0.1:1787760782.298249:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303bc0.
00000080:00000010:0.1:1787760782.298277:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5080.
00010000:00000010:0.1:1787760782.467184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4bb00.
00010000:00000010:0.1:1787760782.467199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6afe00.
00000080:00000010:0.1:1787760782.511457:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303bc0.
00000080:00000010:0.1:1787760782.511481:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc7a0.
00010000:00000010:2.1:1787760782.675312:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4bf00.
00010000:00000010:2.1:1787760782.675322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6af800.
00000080:00000010:2.1:1787760782.728682:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5c80.
00000080:00000010:2.1:1787760782.728696:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc7e0.
00010000:00000010:0.1:1787760782.870192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4bf00.
00010000:00000010:0.1:1787760782.870207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6aec00.
00000080:00000010:0.1:1787760782.893024:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303bc0.
00000080:00000010:0.1:1787760782.893045:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc180.
00010000:00000010:2.1:1787760783.062184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3908500.
00010000:00000010:2.1:1787760783.062200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6ae200.
00000080:00000010:2.1:1787760783.091367:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5ca0.
00000080:00000010:2.1:1787760783.091380:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc0a0.
00010000:00000010:0.1:1787760783.298506:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3908d00.
00010000:00000010:0.1:1787760783.298531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6af200.
00000080:00000010:0.1:1787760783.335235:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303bc0.
00000080:00000010:0.1:1787760783.335248:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc320.
00010000:00000010:0.1:1787760783.464374:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183c00.
00010000:00000010:0.1:1787760783.464387:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468000.
00000080:00000010:0.1:1787760783.492370:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303640.
00000080:00000010:0.1:1787760783.492388:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc4e0.
00010000:00000010:3.1:1787760783.650347:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183f00.
00010000:00000010:3.1:1787760783.650361:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468000.
00000080:00000010:0.1:1787760783.662286:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca8e0.
00000080:00000010:0.1:1787760783.662306:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5da0.
00010000:00000010:0.1:1787760783.797349:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3908800.
00010000:00000010:0.1:1787760783.797359:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6ae600.
00000080:00000010:0.1:1787760783.811612:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303640.
00000080:00000010:0.1:1787760783.811632:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc7e0.
00010000:00000010:0.1:1787760783.941818:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3908f00.
00010000:00000010:0.1:1787760783.941837:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e200.
00000080:00000010:0.1:1787760783.987293:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303860.
00000080:00000010:0.1:1787760783.987308:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303640.
00010000:00000010:0.1:1787760784.180250:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5500.
00010000:00000010:0.1:1787760784.180265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e200.
00000080:00000010:1.1:1787760784.182361:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303860.
00000080:00000010:1.1:1787760784.182385:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303640.
00010000:00000010:2.1:1787760784.348879:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183100.
00010000:00000010:2.1:1787760784.348888:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468000.
00000080:00000010:0.1:1787760784.372836:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df56e0.
00000080:00000010:0.1:1787760784.372859:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5ca0.
00010000:00000010:1.1:1787760784.528386:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3908800.
00010000:00000010:1.1:1787760784.528401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6aec00.
00000080:00000010:2.1:1787760784.560860:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc4e0.
00000080:00000010:2.1:1787760784.560880:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc180.
00010000:00000010:0.1:1787760784.657572:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3908d00.
00010000:00000010:0.1:1787760784.657588:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6aec00.
00000080:00000010:0.1:1787760784.671015:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303c80.
00000080:00000010:0.1:1787760784.671036:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc180.
00010000:00000010:1.1:1787760784.747469:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3908d00.
00010000:00000010:1.1:1787760784.747476:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6afe00.
00000080:00000010:1.1:1787760784.768361:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d83e7cc0.
00000080:00000010:1.1:1787760784.768377:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc180.
00010000:00000010:2.1:1787760784.904350:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183500.
00010000:00000010:2.1:1787760784.904365:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe469200.
00000080:00000010:2.1:1787760784.926061:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5e20.
00000080:00000010:2.1:1787760784.926080:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5760.
00010000:00000010:2.1:1787760785.065411:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183000.
00010000:00000010:2.1:1787760785.065426:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783bc00.
00000080:00000010:2.1:1787760785.089770:0:282442:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5e20.
00000080:00000010:2.1:1787760785.089798:0:282442:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5760.
00010000:00000010:1.1:1787760785.255594:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5000.
00010000:00000010:1.1:1787760785.255610:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e200.
00000080:00000010:1.1:1787760785.304300:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d83e7cc0.
00000080:00000010:1.1:1787760785.304312:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303c80.
00010000:00000010:2.1:1787760785.401125:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5e00.
00010000:00000010:2.1:1787760785.401149:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e800.
00000080:00000010:3.1:1787760785.408127:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5060.
00000080:00000010:3.1:1787760785.408157:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303aa0.
00010000:00000010:2.1:1787760785.508073:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5100.
00010000:00000010:2.1:1787760785.508088:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814800.
00000080:00000010:0.1:1787760785.524173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5500.
00000080:00000010:0.1:1787760785.524192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca060.
00010000:00000010:0.1:1787760785.679242:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3908d00.
00010000:00000010:0.1:1787760785.679252:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6afe00.
00000080:00000010:0.1:1787760785.692039:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303b40.
00000080:00000010:0.1:1787760785.692052:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303740.
00010000:00000010:2.1:1787760785.807182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5b00.
00010000:00000010:2.1:1787760785.807196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e800.
00000080:00000010:0.1:1787760785.852389:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5ce0.
00000080:00000010:0.1:1787760785.852408:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303740.
00010000:00000010:1.1:1787760786.002201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5900.
00010000:00000010:1.1:1787760786.002216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e400.
00000080:00000010:1.1:1787760786.005830:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d83e7cc0.
00000080:00000010:1.1:1787760786.005849:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303740.
00010000:00000010:0.1:1787760786.104189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5400.
00010000:00000010:0.1:1787760786.104204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783b200.
00000080:00000010:0.1:1787760786.136254:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303c80.
00000080:00000010:0.1:1787760786.136268:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df57a0.
00010000:00000010:1.1:1787760786.287287:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5400.
00010000:00000010:1.1:1787760786.287305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783b400.
00000080:00000010:1.1:1787760786.299258:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d83e7cc0.
00000080:00000010:1.1:1787760786.299273:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5a60.
00010000:00000010:0.1:1787760786.415180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5700.
00010000:00000010:0.1:1787760786.415193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e400.
00000080:00000010:0.1:1787760786.432616:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303860.
00000080:00000010:0.1:1787760786.432636:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303c80.
00010000:00000010:0.1:1787760786.529707:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4e17000.
00010000:00000010:0.1:1787760786.529719:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6aee00.
00000080:00000010:0.1:1787760786.540183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303c80.
00000080:00000010:0.1:1787760786.540199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d83e7cc0.
00010000:00000010:0.1:1787760786.672485:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5d00.
00010000:00000010:0.1:1787760786.672498:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da514800.
00000080:00000010:0.1:1787760786.690251:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303de0.
00000080:00000010:0.1:1787760786.690272:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303c80.
00010000:00000010:0.1:1787760786.881365:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5d00.
00010000:00000010:0.1:1787760786.881381:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783a600.
00000080:00000010:0.1:1787760786.915176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303c80.
00000080:00000010:0.1:1787760786.915188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5540.
00010000:00000010:2.1:1787760787.113163:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4e17500.
00010000:00000010:2.1:1787760787.113170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6ae600.
00000080:00000010:0.1:1787760787.141152:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df51a0.
00000080:00000010:0.1:1787760787.141415:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750f7b56140.
00010000:00000010:1.1:1787760787.242238:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5400.
00010000:00000010:1.1:1787760787.242250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da515e00.
00000080:00000010:1.1:1787760787.268177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750f7b56140.
00000080:00000010:1.1:1787760787.268193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303de0.
00010000:00000010:2.1:1787760787.477179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183700.
00010000:00000010:2.1:1787760787.477185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6ae600.
00000080:00000010:1.1:1787760787.490258:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5460.
00000080:00000010:1.1:1787760787.490278:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93030c0.
00010000:00000010:1.1:1787760787.606468:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5b00.
00010000:00000010:1.1:1787760787.606482:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6ae600.
00000080:00000010:0.1:1787760787.641788:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c137a9e0.
00000080:00000010:0.1:1787760787.641806:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0c05f40.
00010000:00000010:1.1:1787760787.823262:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5000.
00010000:00000010:1.1:1787760787.823270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da514a00.
00000080:00000010:1.1:1787760787.833464:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c137a560.
00000080:00000010:1.1:1787760787.833477:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303de0.
00010000:00000010:3.1:1787760787.932188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183200.
00010000:00000010:3.1:1787760787.932205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783b200.
00000080:00000010:0.1:1787760787.964801:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca560.
00000080:00000010:0.1:1787760787.964933:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5460.
00010000:00000010:2.1:1787760788.118282:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183800.
00010000:00000010:2.1:1787760788.118298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783b400.
00000080:00000010:2.1:1787760788.134180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5da0.
00000080:00000010:2.1:1787760788.134195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df57a0.
00010000:00000010:1.1:1787760788.267274:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5b00.
00010000:00000010:1.1:1787760788.267287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da514400.
00000080:00000010:1.1:1787760788.272258:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c137a560.
00000080:00000010:1.1:1787760788.272274:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303de0.
00010000:00000010:3.1:1787760788.419324:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183200.
00010000:00000010:3.1:1787760788.419339:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783a800.
00000080:00000010:0.1:1787760788.455308:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0caa40.
00000080:00000010:0.1:1787760788.455323:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5a60.
00010000:00000010:2.1:1787760788.586370:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5b00.
00010000:00000010:2.1:1787760788.586387:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da515200.
00000080:00000010:0.1:1787760788.606209:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5540.
00000080:00000010:0.1:1787760788.606230:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0c05f40.
00010000:00000010:0.1:1787760788.734481:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d900.
00010000:00000010:0.1:1787760788.734506:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da515a00.
00000080:00000010:0.1:1787760788.744177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c14dc100.
00000080:00000010:0.1:1787760788.744191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c14dcb20.
00010000:00000010:1.1:1787760788.866189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183f00.
00010000:00000010:1.1:1787760788.866203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783be00.
00000080:00000010:2.1:1787760788.879172:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c137a560.
00000080:00000010:2.1:1787760788.879184:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df56e0.
00010000:00000010:0.1:1787760789.011157:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4e17d00.
00010000:00000010:0.1:1787760789.011173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6ae600.
00000080:00000010:0.1:1787760789.041252:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c14dcb20.
00000080:00000010:0.1:1787760789.041272:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c137ae40.
00010000:00000010:2.1:1787760789.270855:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4e17300.
00010000:00000010:2.1:1787760789.270871:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a6000.
00000080:00000010:0.1:1787760789.292502:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5060.
00000080:00000010:0.1:1787760789.292523:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c137a5e0.
00010000:00000010:1.1:1787760789.489191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4e17d00.
00010000:00000010:1.1:1787760789.489210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a7e00.
00000080:00000010:1.1:1787760789.521062:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c137ae40.
00000080:00000010:1.1:1787760789.521082:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c137a5e0.
00010000:00000010:2.1:1787760789.681189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183c00.
00010000:00000010:2.1:1787760789.681204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da515c00.
00000080:00000010:2.1:1787760789.707184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5e40.
00000080:00000010:2.1:1787760789.707204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c14dc100.
00010000:00000010:0.1:1787760789.787510:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3df00.
00010000:00000010:0.1:1787760789.787524:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da515800.
00000080:00000010:0.1:1787760789.799171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0c05f40.
00000080:00000010:0.1:1787760789.799187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c14dc100.
00010000:00000010:1.1:1787760789.906180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4e17500.
00010000:00000010:1.1:1787760789.906195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a6c00.
00000080:00000010:1.1:1787760789.926243:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c137a560.
00000080:00000010:1.1:1787760789.926260:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c137ae40.
00010000:00000010:2.1:1787760790.106501:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3da00.
00010000:00000010:2.1:1787760790.106519:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da515c00.
00000080:00000010:2.1:1787760790.119505:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5080.
00000080:00000010:2.1:1787760790.119520:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5c60.
00010000:00000010:1.1:1787760790.224330:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4e17000.
00010000:00000010:1.1:1787760790.224338:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a6a00.
00000080:00000010:1.1:1787760790.235321:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c137a9e0.
00000080:00000010:1.1:1787760790.235339:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c137a560.
00010000:00000010:0.1:1787760790.346189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1f00.
00010000:00000010:0.1:1787760790.346204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a6200.
00000080:00000010:0.1:1787760790.388303:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c59e0.
00000080:00000010:0.1:1787760790.388322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c137a560.
00010000:00000010:0.1:1787760790.560594:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1200.
00010000:00000010:0.1:1787760790.560611:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a6400.
00000080:00000010:1.1:1787760790.578508:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c59e0.
00000080:00000010:1.1:1787760790.578527:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c137a560.
00010000:00000010:3.1:1787760790.796221:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1a00.
00010000:00000010:3.1:1787760790.796237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a6e00.
00000080:00000010:3.1:1787760790.814193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05400.
00000080:00000010:2.1:1787760790.814507:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05e60.
00010000:00000010:1.1:1787760790.986252:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1300.
00010000:00000010:1.1:1787760790.986274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a7a00.
00000080:00000010:1.1:1787760791.030204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05740.
00000080:00000010:1.1:1787760791.030230:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05e80.
00010000:00000010:0.1:1787760791.255673:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3db00.
00010000:00000010:0.1:1787760791.255697:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da514400.
00000080:00000010:1.1:1787760791.314601:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5d00.
00000080:00000010:1.1:1787760791.314629:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c54e0.
00010000:00000010:2.1:1787760791.544290:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3db00.
00010000:00000010:2.1:1787760791.544313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a7000.
00000080:00000010:0.1:1787760791.574182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df57a0.
00000080:00000010:0.1:1787760791.594283:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05740.
00010000:00000010:0.1:1787760791.809195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1600.
00010000:00000010:0.1:1787760791.809217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a6600.
00000080:00000010:0.1:1787760791.850198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c55c0.
00000080:00000010:0.1:1787760791.850216:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05780.
00010000:00000010:1.1:1787760792.055281:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d500.
00010000:00000010:1.1:1787760792.055298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da514a00.
00000080:00000010:1.1:1787760792.115186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f050a0.
00000080:00000010:1.1:1787760792.115203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c55c0.
00010000:00000010:0.1:1787760792.315577:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d400.
00010000:00000010:0.1:1787760792.315601:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da515e00.
00000080:00000010:0.1:1787760792.347182:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5f60.
00000080:00000010:0.1:1787760792.347205:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5740.
00010000:00000010:1.1:1787760792.502431:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d100.
00010000:00000010:1.1:1787760792.502466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da514a00.
00000080:00000010:1.1:1787760792.531055:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f050a0.
00000080:00000010:1.1:1787760792.531090:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5f60.
00010000:00000010:0.1:1787760792.686968:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d300.
00010000:00000010:0.1:1787760792.687003:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da515c00.
00000080:00000010:0.1:1787760792.723241:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5240.
00000080:00000010:0.1:1787760792.723266:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5000.
00010000:00000010:2.1:1787760793.060192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183300.
00010000:00000010:2.1:1787760793.060234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da514a00.
00000080:00000010:0.1:1787760793.081448:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df56a0.
00000080:00000010:0.1:1787760793.081468:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5000.
00010000:00000010:0.1:1787760793.310406:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5b00.
00010000:00000010:0.1:1787760793.310425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814e00.
00000080:00000010:0.1:1787760793.327180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5000.
00000080:00000010:0.1:1787760793.327201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0caa80.
00010000:00000010:1.1:1787760793.492524:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1700.
00010000:00000010:1.1:1787760793.492540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a7600.
00000080:00000010:2.1:1787760793.522187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05ee0.
00000080:00000010:2.1:1787760793.522202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05f60.
00010000:00000010:1.1:1787760793.791183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5d00.
00010000:00000010:1.1:1787760793.791198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814c00.
00000080:00000010:1.1:1787760793.826201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05480.
00000080:00000010:1.1:1787760793.826221:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca4e0.
00010000:00000010:2.1:1787760794.014718:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d100.
00010000:00000010:2.1:1787760794.014737:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580600.
00000080:00000010:0.1:1787760794.030572:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5400.
00000080:00000010:0.1:1787760794.030593:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5240.
00010000:00000010:2.1:1787760794.252340:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1800.
00010000:00000010:2.1:1787760794.252355:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a6600.
00000080:00000010:2.1:1787760794.262654:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5e40.
00000080:00000010:2.1:1787760794.262673:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05ba0.
00010000:00000010:2.1:1787760794.472426:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d500.
00010000:00000010:2.1:1787760794.472442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581200.
00000080:00000010:2.1:1787760794.490160:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df56a0.
00000080:00000010:2.1:1787760794.490172:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5240.
00010000:00000010:1.1:1787760794.629188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3dd00.
00010000:00000010:1.1:1787760794.629206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581600.
00000080:00000010:1.1:1787760794.643222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f059e0.
00000080:00000010:1.1:1787760794.643235:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c56e0.
00010000:00000010:0.1:1787760794.768733:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5600.
00010000:00000010:0.1:1787760794.768746:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec815600.
00000080:00000010:0.1:1787760794.786175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c52e0.
00000080:00000010:0.1:1787760794.786199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca6a0.
00010000:00000010:3.1:1787760794.947237:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5200.
00010000:00000010:3.1:1787760794.947249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec815a00.
00000080:00000010:3.1:1787760794.982197:0:257571:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca960.
00000080:00000010:3.1:1787760794.982211:0:257571:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cadc0.
00010000:00000010:0.1:1787760795.105585:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3dd00.
00010000:00000010:0.1:1787760795.105598:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580000.
00000080:00000010:0.1:1787760795.121548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c51a0.
00000080:00000010:0.1:1787760795.121566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c52e0.
00010000:00000010:0.1:1787760795.266253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1000.
00010000:00000010:0.1:1787760795.266265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580000.
00000080:00000010:1.1:1787760795.290352:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c51a0.
00000080:00000010:1.1:1787760795.290364:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c52e0.
00010000:00000010:1.1:1787760795.452214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d100.
00010000:00000010:1.1:1787760795.452221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580000.
00000080:00000010:1.1:1787760795.460309:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05cc0.
00000080:00000010:1.1:1787760795.460323:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5dc0.
00010000:00000010:0.1:1787760795.611426:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738100.
00010000:00000010:0.1:1787760795.611442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581e00.
00000080:00000010:1.1:1787760795.634165:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5ea0.
00000080:00000010:1.1:1787760795.634177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5460.
00010000:00000010:1.1:1787760795.753317:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1400.
00010000:00000010:1.1:1787760795.753338:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a6400.
00000080:00000010:1.1:1787760795.769305:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05300.
00000080:00000010:1.1:1787760795.769331:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f051c0.
00010000:00000010:2.1:1787760795.947972:0:257571:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183200.
00010000:00000010:2.1:1787760795.947988:0:257571:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783b800.
00000080:00000010:3.1:1787760795.948448:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5400.
00000080:00000010:3.1:1787760795.948463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5a60.
00010000:00000010:1.1:1787760796.085779:0:1132:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738100.
00010000:00000010:1.1:1787760796.085795:0:1132:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581e00.
00000080:00000010:1.1:1787760796.115187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f051c0.
00000080:00000010:1.1:1787760796.115207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c53c0.
00010000:00000010:0.1:1787760796.294225:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1500.
00010000:00000010:0.1:1787760796.294242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a7e00.
00000080:00000010:0.1:1787760796.327129:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5620.
00000080:00000010:0.1:1787760796.327158:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f051c0.
00010000:00000010:1.1:1787760796.463189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738200.
00010000:00000010:1.1:1787760796.463206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783a600.
00000080:00000010:0.1:1787760796.494252:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05a40.
00000080:00000010:0.1:1787760796.494262:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5e20.
00010000:00000010:0.1:1787760796.619236:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738700.
00010000:00000010:0.1:1787760796.619251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580a00.
00000080:00000010:0.1:1787760796.628280:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5b00.
00000080:00000010:0.1:1787760796.628293:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5860.
00010000:00000010:2.1:1787760796.796299:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738a00.
00010000:00000010:2.1:1787760796.796316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580a00.
00000080:00000010:0.1:1787760796.836323:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5760.
00000080:00000010:0.1:1787760796.836335:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5860.
00010000:00000010:2.1:1787760796.966186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738300.
00010000:00000010:2.1:1787760796.966199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580200.
00000080:00000010:0.1:1787760797.003195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05140.
00000080:00000010:0.1:1787760797.003204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5860.
00010000:00000010:1.1:1787760797.212214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738800.
00010000:00000010:1.1:1787760797.212226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783a400.
00000080:00000010:1.1:1787760797.220207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05460.
00000080:00000010:1.1:1787760797.220222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5540.
00010000:00000010:2.1:1787760797.458582:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738c00.
00010000:00000010:2.1:1787760797.458599:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580400.
00000080:00000010:0.1:1787760797.492678:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5760.
00000080:00000010:0.1:1787760797.492697:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5860.
00010000:00000010:3.1:1787760797.767269:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1900.
00010000:00000010:3.1:1787760797.767292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a7e00.
00000080:00000010:2.1:1787760797.814316:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca960.
00000080:00000010:2.1:1787760797.814342:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05460.
00010000:00000010:1.1:1787760798.034225:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1500.
00010000:00000010:1.1:1787760798.034247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a6200.
00000080:00000010:2.1:1787760798.083683:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f056e0.
00000080:00000010:2.1:1787760798.083713:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05180.
00010000:00000010:3.1:1787760798.325626:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5a00.
00010000:00000010:3.1:1787760798.325645:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec815c00.
00000080:00000010:1.1:1787760798.359689:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cace0.
00000080:00000010:1.1:1787760798.359703:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca7e0.
00010000:00000010:1.1:1787760798.542186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1000.
00010000:00000010:1.1:1787760798.542202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a6600.
00000080:00000010:1.1:1787760798.562166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05100.
00000080:00000010:1.1:1787760798.562175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f050c0.
00010000:00000010:1.1:1787760798.697295:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1800.
00010000:00000010:1.1:1787760798.697305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a7e00.
00000080:00000010:3.1:1787760798.701527:0:257571:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05100.
00000080:00000010:3.1:1787760798.701541:0:257571:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f050c0.
00010000:00000010:2.1:1787760798.889267:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1600.
00010000:00000010:2.1:1787760798.889283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a7e00.
00000080:00000010:0.1:1787760798.905183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5760.
00000080:00000010:0.1:1787760798.905197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f056c0.
00010000:00000010:1.1:1787760799.119177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1200.
00010000:00000010:1.1:1787760799.119195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e9000.
00000080:00000010:1.1:1787760799.156181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05160.
00000080:00000010:1.1:1787760799.156198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f059a0.
00010000:00000010:0.1:1787760799.278714:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738500.
00010000:00000010:0.1:1787760799.278731:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581a00.
00000080:00000010:0.1:1787760799.295447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c58a0.
00000080:00000010:0.1:1787760799.295459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5c40.
00010000:00000010:1.1:1787760799.433180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738500.
00010000:00000010:1.1:1787760799.433194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e8800.
00000080:00000010:1.1:1787760799.450316:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05fa0.
00000080:00000010:1.1:1787760799.450332:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05160.
00010000:00000010:0.1:1787760799.598363:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1f00.
00010000:00000010:0.1:1787760799.598377:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581000.
00000080:00000010:0.1:1787760799.644396:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5880.
00000080:00000010:0.1:1787760799.644407:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c58a0.
00010000:00000010:0.1:1787760799.887524:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738400.
00010000:00000010:0.1:1787760799.887533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581000.
00000080:00000010:0.1:1787760799.912363:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5880.
00000080:00000010:0.1:1787760799.912381:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c58a0.
00010000:00000010:1.1:1787760800.080309:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1600.
00010000:00000010:1.1:1787760800.080325:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e8600.
00000080:00000010:1.1:1787760800.105199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05440.
00000080:00000010:1.1:1787760800.105214:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05fa0.
00010000:00000010:2.1:1787760800.389172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738500.
00010000:00000010:2.1:1787760800.389183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581000.
00000080:00000010:0.1:1787760800.423248:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9980.
00000080:00000010:0.1:1787760800.423265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c58a0.
00010000:00000010:2.1:1787760800.594192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738c00.
00010000:00000010:2.1:1787760800.594209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec815c00.
00000080:00000010:0.1:1787760800.638201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9020.
00000080:00000010:0.1:1787760800.638217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cac20.
00010000:00000010:2.1:1787760800.796182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738300.
00010000:00000010:2.1:1787760800.796196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843e200.
00000080:00000010:0.1:1787760800.821183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e93a0.
00000080:00000010:0.1:1787760800.821197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9fe0.
00010000:00000010:0.1:1787760800.992268:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738200.
00010000:00000010:0.1:1787760800.992281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580e00.
00000080:00000010:0.1:1787760801.021969:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5920.
00000080:00000010:0.1:1787760801.021989:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c59a0.
00010000:00000010:2.1:1787760801.210331:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738100.
00010000:00000010:2.1:1787760801.210356:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580400.
00000080:00000010:0.1:1787760801.226173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9e00.
00000080:00000010:0.1:1787760801.226187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c59a0.
00010000:00000010:0.1:1787760801.396188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738300.
00010000:00000010:0.1:1787760801.396204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580a00.
00000080:00000010:0.1:1787760801.424217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c52c0.
00000080:00000010:0.1:1787760801.424234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c59a0.
00010000:00000010:2.1:1787760801.629191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5600.
00010000:00000010:2.1:1787760801.629207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814c00.
00000080:00000010:0.1:1787760801.641485:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9e80.
00000080:00000010:0.1:1787760801.641497:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca780.
00010000:00000010:0.1:1787760801.799179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47900.
00010000:00000010:0.1:1787760801.799194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e9200.
00000080:00000010:0.1:1787760801.809650:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c59a0.
00000080:00000010:0.1:1787760801.809661:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05440.
00010000:00000010:1.1:1787760802.008186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738c00.
00010000:00000010:1.1:1787760802.008203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580a00.
00000080:00000010:1.1:1787760802.024388:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05ae0.
00000080:00000010:1.1:1787760802.024399:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c59a0.
00010000:00000010:1.1:1787760802.267965:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738c00.
00010000:00000010:1.1:1787760802.267982:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581e00.
00000080:00000010:0.1:1787760802.304441:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05ae0.
00000080:00000010:0.1:1787760802.304457:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c52c0.
00010000:00000010:1.1:1787760802.549187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4a00.
00010000:00000010:1.1:1787760802.549197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843ec00.
00000080:00000010:2.1:1787760802.570995:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f056a0.
00000080:00000010:2.1:1787760802.571014:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9e80.
00010000:00000010:1.1:1787760802.746195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b100.
00010000:00000010:1.1:1787760802.746223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581600.
00000080:00000010:1.1:1787760802.768303:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05800.
00000080:00000010:1.1:1787760802.768331:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c52c0.
00010000:00000010:0.1:1787760802.954387:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b000.
00010000:00000010:0.1:1787760802.954402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580600.
00000080:00000010:0.1:1787760802.961173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c57c0.
00000080:00000010:0.1:1787760802.961188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5140.
00010000:00000010:2.1:1787760803.107499:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47100.
00010000:00000010:2.1:1787760803.107510:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e9600.
00000080:00000010:2.1:1787760803.129568:0:257570:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e93a0.
00000080:00000010:2.1:1787760803.129586:0:257570:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05800.
00010000:00000010:2.1:1787760803.304661:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5d00.
00010000:00000010:2.1:1787760803.304674:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814c00.
00000080:00000010:3.1:1787760803.315180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9f60.
00000080:00000010:3.1:1787760803.315199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9fe0.
00010000:00000010:2.1:1787760803.521559:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b000.
00010000:00000010:2.1:1787760803.521573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843ec00.
00000080:00000010:0.1:1787760803.568167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9de0.
00000080:00000010:0.1:1787760803.568182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9ca0.
00010000:00000010:1.1:1787760803.770874:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47f00.
00010000:00000010:1.1:1787760803.770889:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581600.
00000080:00000010:1.1:1787760803.779524:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05c40.
00000080:00000010:1.1:1787760803.779538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c57c0.
00010000:00000010:0.1:1787760803.967060:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b600.
00010000:00000010:0.1:1787760803.967069:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581e00.
00000080:00000010:2.1:1787760803.989594:0:257571:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5480.
00000080:00000010:2.1:1787760803.989615:0:257571:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c56a0.
00010000:00000010:1.1:1787760804.116297:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b300.
00010000:00000010:1.1:1787760804.116304:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581e00.
00000080:00000010:1.1:1787760804.138175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05c40.
00000080:00000010:1.1:1787760804.138187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5b40.
00010000:00000010:3.1:1787760804.365182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841bb00.
00010000:00000010:3.1:1787760804.365197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580400.
00000080:00000010:3.1:1787760804.379868:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca0e0.
00000080:00000010:3.1:1787760804.379883:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c55a0.
00010000:00000010:0.1:1787760804.596163:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5000.
00010000:00000010:0.1:1787760804.596170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783a600.
00000080:00000010:0.1:1787760804.623167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5ba0.
00000080:00000010:0.1:1787760804.623183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca640.
00010000:00000010:0.1:1787760804.787331:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47f00.
00010000:00000010:0.1:1787760804.787355:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e8000.
00000080:00000010:0.1:1787760804.805221:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5ba0.
00000080:00000010:0.1:1787760804.805233:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05c40.
00010000:00000010:2.1:1787760804.975278:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5000.
00010000:00000010:2.1:1787760804.975293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa38000.
00000080:00000010:1.1:1787760804.992642:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9fe0.
00000080:00000010:1.1:1787760804.992660:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca940.
00010000:00000010:0.1:1787760805.146275:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47b00.
00010000:00000010:0.1:1787760805.146284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e9e00.
00000080:00000010:0.1:1787760805.186187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5ba0.
00000080:00000010:0.1:1787760805.186198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f052e0.
00010000:00000010:0.1:1787760805.361282:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47200.
00010000:00000010:0.1:1787760805.361297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e8a00.
00000080:00000010:0.1:1787760805.369364:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5ba0.
00000080:00000010:0.1:1787760805.369385:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05800.
00010000:00000010:0.1:1787760805.553186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47700.
00010000:00000010:0.1:1787760805.553202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e9800.
00000080:00000010:0.1:1787760805.577186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5ba0.
00000080:00000010:0.1:1787760805.577203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05ae0.
00010000:00000010:3.1:1787760805.842917:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4e00.
00010000:00000010:3.1:1787760805.842931:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8771e00.
00000080:00000010:0.1:1787760805.852320:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cab60.
00000080:00000010:0.1:1787760805.852329:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9600.
00010000:00000010:3.1:1787760806.045198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4800.
00010000:00000010:3.1:1787760806.045218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8770200.
00000080:00000010:0.1:1787760806.072918:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cae60.
00000080:00000010:0.1:1787760806.072932:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9ca0.
00010000:00000010:0.1:1787760806.284104:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841bf00.
00010000:00000010:0.1:1787760806.284112:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783a600.
00000080:00000010:2.1:1787760806.291844:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c50e0.
00000080:00000010:2.1:1787760806.291858:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5ee0.
00010000:00000010:1.1:1787760806.491685:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4300.
00010000:00000010:1.1:1787760806.491694:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8771200.
00000080:00000010:1.1:1787760806.507522:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05440.
00000080:00000010:1.1:1787760806.507541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9de0.
00010000:00000010:2.1:1787760806.677210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47e00.
00010000:00000010:2.1:1787760806.677225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e9400.
00000080:00000010:3.1:1787760806.691472:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9980.
00000080:00000010:3.1:1787760806.691486:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05440.
00010000:00000010:1.1:1787760806.891291:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47000.
00010000:00000010:1.1:1787760806.891314:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e8e00.
00000080:00000010:1.1:1787760806.909318:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f050c0.
00000080:00000010:1.1:1787760806.909348:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5560.
00010000:00000010:0.1:1787760807.136374:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b500.
00010000:00000010:0.1:1787760807.136398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa39200.
00000080:00000010:0.1:1787760807.147468:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5560.
00000080:00000010:0.1:1787760807.147481:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca7c0.
00010000:00000010:2.1:1787760807.385177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47d00.
00010000:00000010:2.1:1787760807.385192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e9a00.
00000080:00000010:2.1:1787760807.432317:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9980.
00000080:00000010:2.1:1787760807.432330:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05100.
00010000:00000010:2.1:1787760807.681182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b500.
00010000:00000010:2.1:1787760807.681191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa39400.
00000080:00000010:0.1:1787760807.697203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9600.
00000080:00000010:0.1:1787760807.697222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cab40.
00010000:00000010:0.1:1787760807.812317:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4e00.
00010000:00000010:0.1:1787760807.812336:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068ba00.
00000080:00000010:0.1:1787760807.822490:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c50e0.
00000080:00000010:0.1:1787760807.822500:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5ee0.
00010000:00000010:0.1:1787760807.998381:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47c00.
00010000:00000010:0.1:1787760807.998392:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e9a00.
00000080:00000010:1.1:1787760808.025927:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5ee0.
00000080:00000010:1.1:1787760808.025947:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05180.
00010000:00000010:2.1:1787760808.204181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47d00.
00010000:00000010:2.1:1787760808.204196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e9800.
00000080:00000010:2.1:1787760808.226476:0:257571:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9fe0.
00000080:00000010:2.1:1787760808.226495:0:257571:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05180.
00010000:00000010:1.1:1787760808.381834:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b400.
00010000:00000010:1.1:1787760808.381850:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068b000.
00000080:00000010:1.1:1787760808.421629:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05460.
00000080:00000010:1.1:1787760808.421644:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c50e0.
00010000:00000010:2.1:1787760808.601505:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4a00.
00010000:00000010:2.1:1787760808.601519:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8771000.
00000080:00000010:3.1:1787760808.617446:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9980.
00000080:00000010:3.1:1787760808.617460:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9e00.
00010000:00000010:2.1:1787760808.827200:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b500.
00010000:00000010:2.1:1787760808.827216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068a800.
00000080:00000010:3.1:1787760808.866007:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9f60.
00000080:00000010:3.1:1787760808.866024:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5b40.
00010000:00000010:0.1:1787760809.072185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5600.
00010000:00000010:0.1:1787760809.072200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa38400.
00000080:00000010:0.1:1787760809.088363:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c56a0.
00000080:00000010:0.1:1787760809.091389:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cae80.
00010000:00000010:2.1:1787760809.334186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47200.
00010000:00000010:2.1:1787760809.334200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e8000.
00000080:00000010:1.1:1787760809.390169:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9e00.
00000080:00000010:1.1:1787760809.390186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05140.
00010000:00000010:0.1:1787760809.674181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5000.
00010000:00000010:0.1:1787760809.674195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa38e00.
00000080:00000010:0.1:1787760809.699240:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c56a0.
00000080:00000010:0.1:1787760809.699257:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0caf00.
00010000:00000010:1.1:1787760809.983445:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b300.
00010000:00000010:1.1:1787760809.983462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068a600.
00000080:00000010:1.1:1787760810.001398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05a40.
00000080:00000010:1.1:1787760810.001415:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c56a0.
00010000:00000010:0.1:1787760810.234180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b100.
00010000:00000010:0.1:1787760810.234195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068b600.
00000080:00000010:2.1:1787760810.294282:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c52c0.
00000080:00000010:2.1:1787760810.294297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c57c0.
00010000:00000010:2.1:1787760810.557511:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b300.
00010000:00000010:2.1:1787760810.557520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068b600.
00000080:00000010:0.1:1787760810.558853:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9e00.
00000080:00000010:0.1:1787760810.558867:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c59a0.
00010000:00000010:1.1:1787760810.745186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47f00.
00010000:00000010:1.1:1787760810.745200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e8000.
00000080:00000010:1.1:1787760810.794234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f055e0.
00000080:00000010:1.1:1787760810.794251:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f051c0.
00010000:00000010:0.1:1787760810.971010:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b300.
00010000:00000010:0.1:1787760810.971021:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068a000.
00000080:00000010:0.1:1787760810.994095:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5860.
00000080:00000010:0.1:1787760810.994108:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c59a0.
00010000:00000010:3.1:1787760811.192473:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27b00.
00010000:00000010:3.1:1787760811.192487:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa39600.
00000080:00000010:3.1:1787760811.192492:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cad60.
00000080:00000010:3.1:1787760811.192501:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca620.
00010000:00000010:3.1:1787760811.393120:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4a00.
00010000:00000010:3.1:1787760811.393151:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068bc00.
00000080:00000010:3.1:1787760811.413807:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca620.
00000080:00000010:3.1:1787760811.413827:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5860.
00010000:00000010:1.1:1787760811.604573:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27900.
00010000:00000010:1.1:1787760811.604587:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa39600.
00000080:00000010:1.1:1787760811.624337:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f051c0.
00000080:00000010:1.1:1787760811.624352:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca620.
00010000:00000010:2.1:1787760811.904188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb900.
00010000:00000010:2.1:1787760811.904201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068be00.
00000080:00000010:0.1:1787760811.910185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9f60.
00000080:00000010:0.1:1787760811.910208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c53c0.
00010000:00000010:2.1:1787760812.088180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb100.
00010000:00000010:2.1:1787760812.088197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068aa00.
00000080:00000010:0.1:1787760812.135546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9f60.
00000080:00000010:0.1:1787760812.135568:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c53c0.
00010000:00000010:1.1:1787760812.243757:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4a00.
00010000:00000010:1.1:1787760812.243767:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8770000.
00000080:00000010:1.1:1787760812.268555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05ba0.
00000080:00000010:1.1:1787760812.268572:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d880.
00010000:00000010:0.1:1787760812.494506:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47100.
00010000:00000010:0.1:1787760812.494519:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e8000.
00000080:00000010:1.1:1787760812.520468:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5dc0.
00000080:00000010:1.1:1787760812.520478:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05ba0.
00010000:00000010:2.1:1787760812.790687:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb700.
00010000:00000010:2.1:1787760812.790695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8771200.
00000080:00000010:0.1:1787760812.821434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d680.
00000080:00000010:0.1:1787760812.821454:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05ba0.
00010000:00000010:1.1:1787760813.052269:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47100.
00010000:00000010:1.1:1787760813.052283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068b400.
00000080:00000010:1.1:1787760813.071190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05ee0.
00000080:00000010:1.1:1787760813.071208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c51a0.
00010000:00000010:0.1:1787760813.314159:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb300.
00010000:00000010:0.1:1787760813.314166:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068ae00.
00000080:00000010:0.1:1787760813.341225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05ee0.
00000080:00000010:0.1:1787760813.341244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c56e0.
00010000:00000010:2.1:1787760813.521188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47f00.
00010000:00000010:2.1:1787760813.521204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e9a00.
00000080:00000010:0.1:1787760813.573546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d880.
00000080:00000010:0.1:1787760813.573559:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f050a0.
00010000:00000010:2.1:1787760813.805952:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb800.
00010000:00000010:2.1:1787760813.805966:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068aa00.
00000080:00000010:0.1:1787760813.832883:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d880.
00000080:00000010:0.1:1787760813.832893:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5f60.
00010000:00000010:1.1:1787760814.048176:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47200.
00010000:00000010:1.1:1787760814.048190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068bc00.
00000080:00000010:1.1:1787760814.076183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05780.
00000080:00000010:1.1:1787760814.076200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5f60.
00010000:00000010:1.1:1787760814.280251:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47d00.
00010000:00000010:1.1:1787760814.280267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e8000.
00000080:00000010:1.1:1787760814.325175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05e60.
00000080:00000010:1.1:1787760814.325192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05780.
00010000:00000010:0.1:1787760814.508299:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27500.
00010000:00000010:0.1:1787760814.508323:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa38e00.
00000080:00000010:0.1:1787760814.534022:0:1051:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c54e0.
00000080:00000010:0.1:1787760814.534031:0:1051:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca5c0.
00010000:00000010:1.1:1787760814.639230:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47f00.
00010000:00000010:1.1:1787760814.639252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874ac00.
00000080:00000010:1.1:1787760814.655179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05400.
00000080:00000010:1.1:1787760814.655194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05e60.
00010000:00000010:0.1:1787760814.778234:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fbe00.
00010000:00000010:0.1:1787760814.778257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068b600.
00000080:00000010:0.1:1787760814.798171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5d00.
00000080:00000010:0.1:1787760814.798181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c54e0.
00010000:00000010:2.1:1787760814.965317:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fbf00.
00010000:00000010:2.1:1787760814.965332:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068a600.
00000080:00000010:0.1:1787760815.007201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d680.
00000080:00000010:0.1:1787760815.007213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c54e0.
00010000:00000010:2.1:1787760815.223012:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4a00.
00010000:00000010:2.1:1787760815.223033:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8771000.
00000080:00000010:0.1:1787760815.288176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d4e0.
00000080:00000010:0.1:1787760815.288212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d880.
00010000:00000010:3.1:1787760815.583251:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fbf00.
00010000:00000010:3.1:1787760815.583263:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068a600.
00000080:00000010:3.1:1787760815.608180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca220.
00000080:00000010:3.1:1787760815.608205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5c60.
00010000:00000010:1.1:1787760815.872287:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb700.
00010000:00000010:1.1:1787760815.872303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068bc00.
00000080:00000010:3.1:1787760815.908195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05e60.
00000080:00000010:3.1:1787760815.908214:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5c60.
00010000:00000010:2.1:1787760816.109188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb100.
00010000:00000010:2.1:1787760816.109201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8770200.
00000080:00000010:0.1:1787760816.141384:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d880.
00000080:00000010:0.1:1787760816.141401:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d520.
00010000:00000010:2.1:1787760816.396187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4a00.
00010000:00000010:2.1:1787760816.396212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8771a00.
00000080:00000010:0.1:1787760816.412383:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d400.
00000080:00000010:0.1:1787760816.412400:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d5a0.
00010000:00000010:1.1:1787760816.599181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c500.
00010000:00000010:1.1:1787760816.599193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874aa00.
00000080:00000010:1.1:1787760816.620190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05e60.
00000080:00000010:1.1:1787760816.620208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05bc0.
00010000:00000010:0.1:1787760816.795187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb700.
00010000:00000010:0.1:1787760816.795201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068bc00.
00000080:00000010:1.1:1787760816.820196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5dc0.
00000080:00000010:1.1:1787760816.820207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c51a0.
00010000:00000010:0.1:1787760817.049664:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb700.
00010000:00000010:0.1:1787760817.049680:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068bc00.
00000080:00000010:0.1:1787760817.062384:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c57c0.
00000080:00000010:0.1:1787760817.062399:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5860.
00010000:00000010:2.1:1787760817.217270:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27a00.
00010000:00000010:2.1:1787760817.217283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa39400.
00000080:00000010:1.1:1787760817.253183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d5a0.
00000080:00000010:1.1:1787760817.253197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca380.
00010000:00000010:1.1:1787760817.387472:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb700.
00010000:00000010:1.1:1787760817.387480:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068bc00.
00000080:00000010:1.1:1787760817.413186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05ee0.
00000080:00000010:1.1:1787760817.413207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5860.
00010000:00000010:2.1:1787760817.564852:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb700.
00010000:00000010:2.1:1787760817.564867:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068bc00.
00000080:00000010:2.1:1787760817.591341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d5a0.
00000080:00000010:2.1:1787760817.591371:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c52c0.
00010000:00000010:0.1:1787760817.888623:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb700.
00010000:00000010:0.1:1787760817.888635:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa38000.
00000080:00000010:0.1:1787760817.905806:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c56a0.
00000080:00000010:0.1:1787760817.905823:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cae00.
00010000:00000010:3.1:1787760818.133169:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c400.
00010000:00000010:3.1:1787760818.133178:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874b400.
00000080:00000010:3.1:1787760818.142185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cae00.
00000080:00000010:3.1:1787760818.142201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05ba0.
00010000:00000010:2.1:1787760818.389327:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c100.
00010000:00000010:2.1:1787760818.389338:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068bc00.
00000080:00000010:0.1:1787760818.397492:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d880.
00000080:00000010:0.1:1787760818.397507:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c56a0.
00010000:00000010:1.1:1787760818.661926:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27300.
00010000:00000010:1.1:1787760818.661938:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8770c00.
00000080:00000010:1.1:1787760818.688495:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f051c0.
00000080:00000010:1.1:1787760818.688513:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d5a0.
00010000:00000010:0.1:1787760818.858121:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35dc00.
00010000:00000010:0.1:1787760818.858154:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068bc00.
00000080:00000010:1.1:1787760818.900507:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5b40.
00000080:00000010:1.1:1787760818.900527:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c56a0.
00010000:00000010:1.1:1787760819.113290:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2dc00.
00010000:00000010:1.1:1787760819.113299:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874b000.
00000080:00000010:1.1:1787760819.134976:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05100.
00000080:00000010:1.1:1787760819.134987:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05180.
00010000:00000010:0.1:1787760819.320939:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2de00.
00010000:00000010:0.1:1787760819.320948:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874a800.
00000080:00000010:0.1:1787760819.352278:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c50e0.
00000080:00000010:0.1:1787760819.352297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05180.
00010000:00000010:0.1:1787760819.511221:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2de00.
00010000:00000010:0.1:1787760819.511236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874b600.
00000080:00000010:0.1:1787760819.525279:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c50e0.
00000080:00000010:0.1:1787760819.525295:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05800.
00010000:00000010:0.1:1787760819.788183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35df00.
00010000:00000010:0.1:1787760819.788273:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8770000.
00000080:00000010:0.1:1787760819.816541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c50e0.
00000080:00000010:0.1:1787760819.816557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d5a0.
00010000:00000010:0.1:1787760820.012437:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c600.
00010000:00000010:0.1:1787760820.012449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874bc00.
00000080:00000010:0.1:1787760820.030237:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c50e0.
00000080:00000010:0.1:1787760820.030251:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05800.
00010000:00000010:0.1:1787760820.262466:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2df00.
00010000:00000010:0.1:1787760820.262476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068bc00.
00000080:00000010:1.1:1787760820.282809:0:1365:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5ee0.
00000080:00000010:1.1:1787760820.282956:0:1365:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c50e0.
00010000:00000010:2.1:1787760820.460186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2d500.
00010000:00000010:2.1:1787760820.460211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068bc00.
00000080:00000010:2.1:1787760820.492244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750f9df7aa0.
00000080:00000010:2.1:1787760820.492263:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c50e0.
00010000:00000010:0.1:1787760820.790929:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2dd00.
00010000:00000010:0.1:1787760820.790941:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068bc00.
00000080:00000010:0.1:1787760820.805173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c56a0.
00000080:00000010:0.1:1787760820.805188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5ee0.
00010000:00000010:2.1:1787760820.961179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35d600.
00010000:00000010:2.1:1787760820.961194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8771a00.
00000080:00000010:3.1:1787760820.977413:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750f9df7ba0.
00000080:00000010:3.1:1787760820.977428:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750f9df7e20.
00010000:00000010:1.1:1787760821.167315:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c300.
00010000:00000010:1.1:1787760821.167332:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874be00.
00000080:00000010:0.1:1787760821.212699:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05100.
00000080:00000010:0.1:1787760821.212718:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05180.
00010000:00000010:3.1:1787760821.426475:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2d400.
00010000:00000010:3.1:1787760821.426489:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0400.
00000080:00000010:3.1:1787760821.439309:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cae80.
00000080:00000010:3.1:1787760821.439319:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c56a0.
00010000:00000010:0.1:1787760821.648856:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c300.
00010000:00000010:0.1:1787760821.648870:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874be00.
00000080:00000010:0.1:1787760821.674200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5b40.
00000080:00000010:0.1:1787760821.674223:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05ba0.
00010000:00000010:1.1:1787760821.905368:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2d000.
00010000:00000010:1.1:1787760821.905378:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a1400.
00000080:00000010:1.1:1787760821.926727:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05ba0.
00000080:00000010:1.1:1787760821.926745:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5b40.
00010000:00000010:0.1:1787760822.170502:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c800.
00010000:00000010:0.1:1787760822.170527:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0e00.
00000080:00000010:0.1:1787760822.211788:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5860.
00000080:00000010:0.1:1787760822.211801:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c52c0.
00010000:00000010:1.1:1787760822.415306:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858cc00.
00010000:00000010:1.1:1787760822.415315:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874bc00.
00000080:00000010:1.1:1787760822.438201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05100.
00000080:00000010:1.1:1787760822.438213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05180.
00010000:00000010:1.1:1787760822.596333:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2d600.
00010000:00000010:1.1:1787760822.596340:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a1a00.
00000080:00000010:1.1:1787760822.603270:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05180.
00000080:00000010:1.1:1787760822.603283:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5860.
00010000:00000010:1.1:1787760822.855824:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858cc00.
00010000:00000010:1.1:1787760822.855836:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d7600.
00000080:00000010:1.1:1787760822.890292:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05180.
00000080:00000010:1.1:1787760822.890303:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cae80.
00010000:00000010:3.1:1787760823.147193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35da00.
00010000:00000010:3.1:1787760823.147207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787a600.
00000080:00000010:0.1:1787760823.178223:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cab40.
00000080:00000010:0.1:1787760823.178236:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750f9df7ba0.
00010000:00000010:2.1:1787760823.436366:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858ce00.
00010000:00000010:2.1:1787760823.436381:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874b600.
00000080:00000010:2.1:1787760823.467405:0:257570:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750f9df7e20.
00000080:00000010:2.1:1787760823.467419:0:257570:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05100.
00010000:00000010:0.1:1787760823.599449:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2d800.
00010000:00000010:0.1:1787760823.599458:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a1000.
00000080:00000010:0.1:1787760823.619176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5dc0.
00000080:00000010:0.1:1787760823.619197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c51a0.
00010000:00000010:1.1:1787760823.811195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27d00.
00010000:00000010:1.1:1787760823.811210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d6000.
00000080:00000010:1.1:1787760823.830270:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05800.
00000080:00000010:1.1:1787760823.830288:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cae60.
00010000:00000010:1.1:1787760824.115512:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2d600.
00010000:00000010:1.1:1787760824.115528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a1000.
00000080:00000010:1.1:1787760824.140189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05800.
00000080:00000010:1.1:1787760824.140204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c51a0.
00010000:00000010:0.1:1787760824.321211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858ce00.
00010000:00000010:0.1:1787760824.321225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874a800.
00000080:00000010:0.1:1787760824.347377:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5dc0.
00000080:00000010:0.1:1787760824.347397:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05800.
00010000:00000010:1.1:1787760824.487423:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c300.
00010000:00000010:1.1:1787760824.487432:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874b400.
00000080:00000010:1.1:1787760824.502376:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c137a560.
00000080:00000010:1.1:1787760824.502398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05800.
00010000:00000010:2.1:1787760824.644368:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c600.
00010000:00000010:2.1:1787760824.644378:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874a800.
00000080:00000010:0.1:1787760824.676285:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202c00.
00000080:00000010:0.1:1787760824.676306:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5280.
00010000:00000010:2.1:1787760824.806211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35db00.
00010000:00000010:2.1:1787760824.806225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787b000.
00000080:00000010:2.1:1787760824.839329:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202880.
00000080:00000010:2.1:1787760824.839347:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202ca0.
00010000:00000010:0.1:1787760825.013185:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c400.
00010000:00000010:0.1:1787760825.013191:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874a800.
00000080:00000010:0.1:1787760825.028707:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5dc0.
00000080:00000010:0.1:1787760825.028724:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b52e0.
00010000:00000010:2.1:1787760825.281193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2d400.
00010000:00000010:2.1:1787760825.281207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0800.
00000080:00000010:2.1:1787760825.297204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202880.
00000080:00000010:2.1:1787760825.297215:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5dc0.
00010000:00000010:0.1:1787760825.485198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2dc00.
00010000:00000010:0.1:1787760825.485212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0600.
00000080:00000010:2.1:1787760825.500509:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5dc0.
00000080:00000010:2.1:1787760825.500527:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5c60.
00010000:00000010:3.1:1787760825.709597:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35d300.
00010000:00000010:3.1:1787760825.709621:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787a600.
00000080:00000010:0.1:1787760825.727510:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca780.
00000080:00000010:0.1:1787760825.727527:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202300.
00010000:00000010:0.1:1787760825.903097:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2d400.
00010000:00000010:0.1:1787760825.903105:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a1200.
00000080:00000010:0.1:1787760825.913249:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5b40.
00000080:00000010:0.1:1787760825.913264:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5860.
00010000:00000010:3.1:1787760826.155198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c600.
00010000:00000010:3.1:1787760826.155213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7898c00.
00000080:00000010:0.1:1787760826.190193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca7e0.
00000080:00000010:0.1:1787760826.190210:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5fa0.
00010000:00000010:2.1:1787760826.393425:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35d100.
00010000:00000010:2.1:1787760826.393439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787a400.
00000080:00000010:2.1:1787760826.406196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202440.
00000080:00000010:2.1:1787760826.406211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202ba0.
00010000:00000010:2.1:1787760826.700919:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858ce00.
00010000:00000010:2.1:1787760826.700930:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7898200.
00000080:00000010:3.1:1787760826.706190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202320.
00000080:00000010:3.1:1787760826.706206:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b50c0.
00010000:00000010:0.1:1787760826.924329:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565e00.
00010000:00000010:0.1:1787760826.924347:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0000.
00000080:00000010:1.1:1787760826.940181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5b40.
00000080:00000010:1.1:1787760826.940193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c50e0.
00010000:00000010:1.1:1787760827.118392:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c600.
00010000:00000010:1.1:1787760827.118408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7898400.
00000080:00000010:3.1:1787760827.144371:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5da0.
00000080:00000010:3.1:1787760827.144381:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5f20.
00010000:00000010:0.1:1787760827.403253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27d00.
00010000:00000010:0.1:1787760827.403263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d7800.
00000080:00000010:0.1:1787760827.446192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5c60.
00000080:00000010:0.1:1787760827.446206:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca960.
00010000:00000010:0.1:1787760827.715186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565f00.
00010000:00000010:0.1:1787760827.715202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a1c00.
00000080:00000010:0.1:1787760827.757177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c50e0.
00000080:00000010:0.1:1787760827.757188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5c60.
00010000:00000010:2.1:1787760827.904555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35d500.
00010000:00000010:2.1:1787760827.904565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787b800.
00000080:00000010:2.1:1787760827.919613:0:257571:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202e80.
00000080:00000010:2.1:1787760827.919634:0:257571:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202320.
00010000:00000010:0.1:1787760828.065452:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27a00.
00010000:00000010:0.1:1787760828.065459:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d6400.
00000080:00000010:0.1:1787760828.080176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5c60.
00000080:00000010:0.1:1787760828.080189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca4e0.
00010000:00000010:2.1:1787760828.231209:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22600.
00010000:00000010:2.1:1787760828.231235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7899400.
00000080:00000010:0.1:1787760828.255226:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202ba0.
00000080:00000010:0.1:1787760828.255241:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b58e0.
00010000:00000010:1.1:1787760828.436300:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22300.
00010000:00000010:1.1:1787760828.436309:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7898e00.
00000080:00000010:2.1:1787760828.446395:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5bc0.
00000080:00000010:2.1:1787760828.446411:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b56c0.
00010000:00000010:0.1:1787760828.721234:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27900.
00010000:00000010:0.1:1787760828.721250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d7a00.
00000080:00000010:0.1:1787760828.739222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c50e0.
00000080:00000010:0.1:1787760828.739236:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0caa80.
00010000:00000010:1.1:1787760828.957203:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27a00.
00010000:00000010:1.1:1787760828.957373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d6800.
00000080:00000010:1.1:1787760828.987182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5240.
00000080:00000010:1.1:1787760828.987192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca560.
00010000:00000010:2.1:1787760829.177192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565500.
00010000:00000010:2.1:1787760829.177209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787b000.
00000080:00000010:0.1:1787760829.204004:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202200.
00000080:00000010:0.1:1787760829.204035:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d52022a0.
00010000:00000010:2.1:1787760829.422187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35d300.
00010000:00000010:2.1:1787760829.422203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7899a00.
00000080:00000010:2.1:1787760829.458173:0:257570:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202320.
00000080:00000010:2.1:1787760829.458183:0:257570:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5380.
00010000:00000010:2.1:1787760829.667193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22800.
00010000:00000010:2.1:1787760829.667220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0c00.
00000080:00000010:3.1:1787760829.708185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202c40.
00000080:00000010:3.1:1787760829.708198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5b40.
00010000:00000010:0.1:1787760829.985355:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565400.
00010000:00000010:0.1:1787760829.985371:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0200.
00000080:00000010:0.1:1787760830.018178:0:1051:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9f60.
00000080:00000010:0.1:1787760830.018189:0:1051:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d5a0.
00010000:00000010:0.1:1787760830.212183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27a00.
00010000:00000010:0.1:1787760830.212198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d7a00.
00000080:00000010:2.1:1787760830.226165:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9f60.
00000080:00000010:2.1:1787760830.226178:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca8e0.
00010000:00000010:0.1:1787760830.419339:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565100.
00010000:00000010:0.1:1787760830.419353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0200.
00000080:00000010:0.1:1787760830.436182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d5a0.
00000080:00000010:0.1:1787760830.436195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9f60.
00010000:00000010:0.1:1787760830.593492:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565200.
00010000:00000010:0.1:1787760830.593507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0200.
00000080:00000010:0.1:1787760830.649219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9f60.
00000080:00000010:0.1:1787760830.649242:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d5a0.
00010000:00000010:0.1:1787760830.805734:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565200.
00010000:00000010:0.1:1787760830.805746:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787a200.
00000080:00000010:0.1:1787760830.842194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9f60.
00000080:00000010:0.1:1787760830.842222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202ca0.
00010000:00000010:1.1:1787760831.044943:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22c00.
00010000:00000010:1.1:1787760831.044960:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7898800.
00000080:00000010:1.1:1787760831.051741:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5a20.
00000080:00000010:1.1:1787760831.051754:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5680.
00010000:00000010:0.1:1787760831.338182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565000.
00010000:00000010:0.1:1787760831.338207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0200.
00000080:00000010:0.1:1787760831.355592:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d5a0.
00000080:00000010:0.1:1787760831.355625:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9f60.
00010000:00000010:1.1:1787760831.661181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22c00.
00010000:00000010:1.1:1787760831.661199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7898800.
00000080:00000010:1.1:1787760831.688284:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5a20.
00000080:00000010:1.1:1787760831.688299:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5680.
00010000:00000010:1.1:1787760832.039329:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565a00.
00010000:00000010:1.1:1787760832.039346:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a1800.
00000080:00000010:0.1:1787760832.070359:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5680.
00000080:00000010:0.1:1787760832.070371:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e59e9f60.
00010000:00000010:2.1:1787760832.485240:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565600.
00010000:00000010:2.1:1787760832.485258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787a800.
00000080:00000010:0.1:1787760832.508749:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202ca0.
00000080:00000010:0.1:1787760832.508771:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202b00.
00010000:00000010:1.1:1787760832.844201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22c00.
00010000:00000010:1.1:1787760832.844226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7898800.
00000080:00000010:1.1:1787760832.861126:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5aa0.
00000080:00000010:1.1:1787760832.861159:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5a20.
00010000:00000010:0.1:1787760833.176334:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565b00.
00010000:00000010:0.1:1787760833.176353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0800.
00000080:00000010:0.1:1787760833.201225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0c05f40.
00000080:00000010:0.1:1787760833.201253:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c14dc100.
00010000:00000010:2.1:1787760833.533188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22c00.
00010000:00000010:2.1:1787760833.533332:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0800.
00000080:00000010:0.1:1787760833.572326:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202e80.
00000080:00000010:0.1:1787760833.572348:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0c05f40.
00010000:00000010:2.1:1787760833.917484:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30700.
00010000:00000010:2.1:1787760833.917519:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa435c00.
00000080:00000010:3.1:1787760833.972582:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202c40.
00000080:00000010:3.1:1787760833.972606:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202320.
00010000:00000010:0.1:1787760834.264300:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35df00.
00010000:00000010:0.1:1787760834.264311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434200.
00000080:00000010:0.1:1787760834.264315:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0c05f40.
00000080:00000010:0.1:1787760834.264324:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202ba0.
00010000:00000010:2.1:1787760834.575063:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565600.
00010000:00000010:2.1:1787760834.575079:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa435000.
00000080:00000010:0.1:1787760834.590165:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202140.
00000080:00000010:0.1:1787760834.590186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202200.
00010000:00000010:2.1:1787760834.976603:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22f00.
00010000:00000010:2.1:1787760834.976629:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7898600.
00000080:00000010:2.1:1787760835.001326:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202320.
00000080:00000010:2.1:1787760835.001344:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5aa0.
00010000:00000010:2.1:1787760835.245186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565500.
00010000:00000010:2.1:1787760835.245198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0400.
00000080:00000010:2.1:1787760835.272662:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202320.
00000080:00000010:2.1:1787760835.272678:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c77b8740.
00010000:00000010:2.1:1787760835.500722:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22d00.
00010000:00000010:2.1:1787760835.500748:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7899600.
00000080:00000010:2.1:1787760835.512177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202ca0.
00000080:00000010:2.1:1787760835.512201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5880.
00010000:00000010:0.1:1787760835.769401:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30f00.
00010000:00000010:0.1:1787760835.769418:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d6200.
00000080:00000010:0.1:1787760835.796195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c77b8740.
00000080:00000010:0.1:1787760835.796217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca960.
00010000:00000010:2.1:1787760835.941972:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22400.
00010000:00000010:2.1:1787760835.941990:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7899c00.
00000080:00000010:1.1:1787760835.953268:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202ca0.
00000080:00000010:1.1:1787760835.953290:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5cc0.
00010000:00000010:2.1:1787760836.128306:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22200.
00010000:00000010:2.1:1787760836.128316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7899c00.
00000080:00000010:2.1:1787760836.137535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202140.
00000080:00000010:2.1:1787760836.137549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5880.
00010000:00000010:1.1:1787760836.379183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565500.
00010000:00000010:1.1:1787760836.379201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d7600.
00000080:00000010:1.1:1787760836.390710:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5cc0.
00000080:00000010:1.1:1787760836.390724:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202140.
00010000:00000010:3.1:1787760836.664248:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30800.
00010000:00000010:3.1:1787760836.664256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d6200.
00000080:00000010:3.1:1787760836.706183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cae60.
00000080:00000010:3.1:1787760836.706197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca780.
00010000:00000010:0.1:1787760837.027797:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565500.
00010000:00000010:0.1:1787760837.027811:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a1a00.
00000080:00000010:0.1:1787760837.099652:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c302b5c0.
00000080:00000010:0.1:1787760837.099668:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751da3914c0.
00010000:00000010:3.1:1787760837.414249:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22000.
00010000:00000010:3.1:1787760837.414265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7899600.
00000080:00000010:2.1:1787760837.421329:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca780.
00000080:00000010:2.1:1787760837.421347:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c302b5c0.
00010000:00000010:1.1:1787760837.724189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6ad00.
00010000:00000010:1.1:1787760837.724208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0800.
00000080:00000010:1.1:1787760837.757300:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5cc0.
00000080:00000010:1.1:1787760837.757321:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e9202520.
00010000:00000010:2.1:1787760838.109170:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30400.
00010000:00000010:2.1:1787760838.109185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d7a00.
00000080:00000010:0.1:1787760838.126239:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202320.
00000080:00000010:0.1:1787760838.126255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cae60.
00010000:00000010:1.1:1787760838.403189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6a400.
00010000:00000010:1.1:1787760838.403203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867a600.
00000080:00000010:1.1:1787760838.418243:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5cc0.
00000080:00000010:1.1:1787760838.418257:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e92022a0.
00010000:00000010:0.1:1787760838.648301:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30b00.
00010000:00000010:0.1:1787760838.648309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d7a00.
00000080:00000010:0.1:1787760838.688193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e9202800.
00000080:00000010:0.1:1787760838.688214:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cae80.
00010000:00000010:0.1:1787760838.946211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22a00.
00010000:00000010:0.1:1787760838.946227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7899a00.
00000080:00000010:0.1:1787760838.986185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e9202800.
00000080:00000010:0.1:1787760838.986206:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5aa0.
00010000:00000010:0.1:1787760839.281279:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46e00.
00010000:00000010:0.1:1787760839.281292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434c00.
00000080:00000010:0.1:1787760839.337262:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e9202800.
00000080:00000010:0.1:1787760839.337279:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202140.
00010000:00000010:1.1:1787760839.571304:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22a00.
00010000:00000010:1.1:1787760839.571318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7899400.
00000080:00000010:3.1:1787760839.612485:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5380.
00000080:00000010:3.1:1787760839.612502:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5680.
00010000:00000010:0.1:1787760839.770179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6a000.
00010000:00000010:0.1:1787760839.770196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867a000.
00000080:00000010:0.1:1787760839.804762:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f138e0.
00000080:00000010:0.1:1787760839.804774:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e9202520.
00010000:00000010:2.1:1787760840.140219:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30e00.
00010000:00000010:2.1:1787760840.140232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434e00.
00000080:00000010:0.1:1787760840.168102:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202320.
00000080:00000010:0.1:1787760840.168258:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202200.
00010000:00000010:0.1:1787760840.434169:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22000.
00010000:00000010:0.1:1787760840.434177:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7898200.
00000080:00000010:0.1:1787760840.466180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f138e0.
00000080:00000010:0.1:1787760840.466214:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5bc0.
00010000:00000010:0.1:1787760840.717175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22400.
00010000:00000010:0.1:1787760840.717185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7898c00.
00000080:00000010:0.1:1787760840.721285:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f138e0.
00000080:00000010:0.1:1787760840.721306:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5f20.
00010000:00000010:1.1:1787760840.912541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22f00.
00010000:00000010:1.1:1787760840.912550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7899a00.
00000080:00000010:1.1:1787760840.937231:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b55c0.
00000080:00000010:1.1:1787760840.937241:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b50c0.
00010000:00000010:0.1:1787760841.147120:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6aa00.
00010000:00000010:0.1:1787760841.147156:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867ac00.
00000080:00000010:0.1:1787760841.183183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f13220.
00000080:00000010:0.1:1787760841.183193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f13100.
00010000:00000010:2.1:1787760841.369456:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6a600.
00010000:00000010:2.1:1787760841.369470:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867aa00.
00000080:00000010:0.1:1787760841.410420:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202200.
00000080:00000010:0.1:1787760841.410433:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f13100.
00010000:00000010:0.1:1787760841.633181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46f00.
00010000:00000010:0.1:1787760841.633195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434000.
00000080:00000010:1.1:1787760841.671175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f13c00.
00000080:00000010:1.1:1787760841.671186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f13100.
00010000:00000010:0.1:1787760841.971191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22c00.
00010000:00000010:0.1:1787760841.971207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7899600.
00000080:00000010:1.1:1787760841.994340:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f131e0.
00000080:00000010:1.1:1787760841.994360:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b55c0.
00010000:00000010:2.1:1787760842.248425:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22600.
00010000:00000010:2.1:1787760842.248442:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7899600.
00000080:00000010:2.1:1787760842.261584:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202320.
00000080:00000010:2.1:1787760842.261611:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b55c0.
00010000:00000010:1.1:1787760842.506397:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22c00.
00010000:00000010:1.1:1787760842.506414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7899600.
00000080:00000010:1.1:1787760842.541299:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f13b00.
00000080:00000010:1.1:1787760842.541325:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5280.
00010000:00000010:2.1:1787760842.813381:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6a800.
00010000:00000010:2.1:1787760842.813396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867b800.
00000080:00000010:2.1:1787760842.828214:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202320.
00000080:00000010:2.1:1787760842.828226:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5280.
00010000:00000010:2.1:1787760843.210255:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6ab00.
00010000:00000010:2.1:1787760843.210270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa435200.
00000080:00000010:2.1:1787760843.242074:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202140.
00000080:00000010:2.1:1787760843.242099:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202320.
00010000:00000010:0.1:1787760843.602333:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46900.
00010000:00000010:0.1:1787760843.602350:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434600.
00000080:00000010:0.1:1787760843.602355:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f13b00.
00000080:00000010:0.1:1787760843.602366:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202140.
00010000:00000010:2.1:1787760843.964368:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46100.
00010000:00000010:2.1:1787760843.964380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa435a00.
00000080:00000010:2.1:1787760844.023244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202140.
00000080:00000010:2.1:1787760844.023253:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202200.
00010000:00000010:2.1:1787760844.246097:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6ae00.
00010000:00000010:2.1:1787760844.246113:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867b400.
00000080:00000010:2.1:1787760844.263701:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750d5202200.
00000080:00000010:2.1:1787760844.263718:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f13b00.
00010000:00000010:0.1:1787760844.549214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46200.
00010000:00000010:0.1:1787760844.549231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434400.
00000080:00000010:0.1:1787760844.615862:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f13100.
00000080:00000010:0.1:1787760844.615882:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5ac0.
00010000:00000010:3.1:1787760844.868820:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46000.
00010000:00000010:3.1:1787760844.868841:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434000.
00000080:00000010:0.1:1787760844.931216:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca560.
00000080:00000010:0.1:1787760844.931235:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d860e0a0.
00010000:00000010:2.1:1787760845.071631:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46a00.
00010000:00000010:2.1:1787760845.071647:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434a00.
00000080:00000010:2.1:1787760845.090337:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c739dc80.
00000080:00000010:2.1:1787760845.090355:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ecd83320.
00010000:00000010:0.1:1787760845.256025:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22400.
00010000:00000010:0.1:1787760845.256051:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867ba00.
00000080:00000010:0.1:1787760845.273272:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f138e0.
00000080:00000010:0.1:1787760845.273297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f13100.
00010000:00000010:1.1:1787760845.418186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6a800.
00010000:00000010:1.1:1787760845.418212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867b000.
00000080:00000010:1.1:1787760845.442231:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5f20.
00000080:00000010:1.1:1787760845.442243:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f13100.
00010000:00000010:1.1:1787760845.669211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6a400.
00010000:00000010:1.1:1787760845.669219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867a800.
00000080:00000010:2.1:1787760845.683159:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5f20.
00000080:00000010:2.1:1787760845.683172:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f138e0.
00010000:00000010:1.1:1787760845.931379:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22000.
00010000:00000010:1.1:1787760845.931400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa52400.
00000080:00000010:0.1:1787760845.989918:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5380.
00000080:00000010:0.1:1787760845.989929:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5680.
00010000:00000010:0.1:1787760846.195324:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30100.
00010000:00000010:0.1:1787760846.195341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aae00.
00000080:00000010:0.1:1787760846.232185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f13100.
00000080:00000010:0.1:1787760846.232202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca560.
00010000:00000010:0.1:1787760846.378174:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6a400.
00010000:00000010:0.1:1787760846.378183:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53400.
00000080:00000010:0.1:1787760846.387644:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f13100.
00000080:00000010:0.1:1787760846.387657:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5880.
00010000:00000010:0.1:1787760846.565223:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6a800.
00010000:00000010:0.1:1787760846.565239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867b400.
00000080:00000010:0.1:1787760846.614178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5ac0.
00000080:00000010:0.1:1787760846.614189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f13100.
00010000:00000010:2.1:1787760846.890193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30a00.
00010000:00000010:2.1:1787760846.890209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aba00.
00000080:00000010:0.1:1787760846.938222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c739d840.
00000080:00000010:0.1:1787760846.938235:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca560.
00010000:00000010:3.1:1787760847.240190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784e300.
00010000:00000010:3.1:1787760847.240202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53a00.
00000080:00000010:3.1:1787760847.276463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c302b5c0.
00000080:00000010:3.1:1787760847.276483:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5680.
00010000:00000010:2.1:1787760847.593265:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784eb00.
00010000:00000010:2.1:1787760847.593282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa52800.
00000080:00000010:0.1:1787760847.613632:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c739d380.
00000080:00000010:0.1:1787760847.613647:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5f20.
00010000:00000010:3.1:1787760847.958190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30900.
00010000:00000010:3.1:1787760847.958205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aa800.
00000080:00000010:3.1:1787760847.987545:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d940.
00000080:00000010:3.1:1787760847.987560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d520.
00010000:00000010:1.1:1787760848.301245:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784ee00.
00010000:00000010:1.1:1787760848.301260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53200.
00000080:00000010:1.1:1787760848.351256:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5280.
00000080:00000010:1.1:1787760848.351273:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5f20.
00010000:00000010:0.1:1787760848.681329:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2800.
00010000:00000010:0.1:1787760848.681346:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867b800.
00000080:00000010:2.1:1787760848.692308:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca560.
00000080:00000010:2.1:1787760848.692322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e1df5ac0.
00010000:00000010:1.1:1787760848.921172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784ee00.
00010000:00000010:1.1:1787760848.921179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aa800.
00000080:00000010:3.1:1787760848.932639:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5f20.
00000080:00000010:3.1:1787760848.932661:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d940.
00010000:00000010:1.1:1787760849.086597:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784ee00.
00010000:00000010:1.1:1787760849.086610:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53200.
00000080:00000010:1.1:1787760849.107179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5f20.
00000080:00000010:1.1:1787760849.107191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5280.
00010000:00000010:0.1:1787760849.287462:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784ef00.
00010000:00000010:0.1:1787760849.287476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53200.
00000080:00000010:0.1:1787760849.305358:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f13100.
00000080:00000010:0.1:1787760849.305377:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5280.
00010000:00000010:1.1:1787760849.473301:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784ed00.
00010000:00000010:1.1:1787760849.473314:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53600.
00000080:00000010:1.1:1787760849.518789:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5280.
00000080:00000010:1.1:1787760849.518799:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5f20.
00010000:00000010:3.1:1787760849.966191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2c00.
00010000:00000010:3.1:1787760849.966212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867bc00.
00000080:00000010:3.1:1787760849.982174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d6a0.
00000080:00000010:3.1:1787760849.982207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e6f13100.
00010000:00000010:1.1:1787760850.311449:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2f00.
00010000:00000010:1.1:1787760850.311465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867bc00.
00000080:00000010:1.1:1787760850.322527:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5f20.
00000080:00000010:1.1:1787760850.322718:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c5522c00.
00010000:00000010:3.1:1787760850.675309:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46600.
00010000:00000010:3.1:1787760850.675322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79ab600.
00000080:00000010:0.1:1787760850.695171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811ddc0.
00000080:00000010:0.1:1787760850.695184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d6a0.
00010000:00000010:3.1:1787760851.078246:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46300.
00010000:00000010:3.1:1787760851.078256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434a00.
00000080:00000010:0.1:1787760851.118209:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d060.
00000080:00000010:0.1:1787760851.118231:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ecd83320.
00010000:00000010:2.1:1787760851.477777:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2d00.
00010000:00000010:2.1:1787760851.477789:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867bc00.
00000080:00000010:3.1:1787760851.489768:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060620.
00000080:00000010:3.1:1787760851.489787:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c5522c00.
00010000:00000010:1.1:1787760851.801305:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2900.
00010000:00000010:1.1:1787760851.801320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434c00.
00000080:00000010:1.1:1787760851.853395:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7b5280.
00000080:00000010:1.1:1787760851.853416:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060380.
00010000:00000010:2.1:1787760852.160187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2400.
00010000:00000010:2.1:1787760852.160202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434c00.
00000080:00000010:0.1:1787760852.203179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060ca0.
00000080:00000010:0.1:1787760852.203195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060740.
00010000:00000010:2.1:1787760852.526179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784e400.
00010000:00000010:2.1:1787760852.526192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434c00.
00000080:00000010:0.1:1787760852.560184:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060740.
00000080:00000010:0.1:1787760852.560197:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060e60.
00010000:00000010:2.1:1787760852.921525:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2000.
00010000:00000010:2.1:1787760852.921541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8760600.
00000080:00000010:0.1:1787760852.954181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060440.
00000080:00000010:0.1:1787760852.954199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ecd83320.
00010000:00000010:2.1:1787760853.275293:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2700.
00010000:00000010:2.1:1787760853.275308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434a00.
00000080:00000010:0.1:1787760853.332399:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060380.
00000080:00000010:0.1:1787760853.332414:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060920.
00010000:00000010:2.1:1787760853.643703:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2a00.
00010000:00000010:2.1:1787760853.643718:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434a00.
00000080:00000010:2.1:1787760853.677159:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060780.
00000080:00000010:2.1:1787760853.677172:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca0604e0.
00010000:00000010:0.1:1787760854.042244:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2600.
00010000:00000010:0.1:1787760854.042260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8761600.
00000080:00000010:0.1:1787760854.079467:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c61aeaa0.
00000080:00000010:0.1:1787760854.079478:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c61aeb20.
00010000:00000010:0.1:1787760854.558249:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2300.
00010000:00000010:0.1:1787760854.558263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8760000.
00000080:00000010:0.1:1787760854.625178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c61ae3c0.
00000080:00000010:0.1:1787760854.625191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c61aeaa0.
00010000:00000010:2.1:1787760854.941702:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2300.
00010000:00000010:2.1:1787760854.941718:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8592200.
00000080:00000010:0.1:1787760854.982642:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060620.
00000080:00000010:0.1:1787760854.982659:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060240.
00010000:00000010:2.1:1787760855.284459:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784e000.
00010000:00000010:2.1:1787760855.284495:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa52c00.
00000080:00000010:0.1:1787760855.346237:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060780.
00000080:00000010:0.1:1787760855.346252:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0ecc440.
00010000:00000010:2.1:1787760855.689191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2300.
00010000:00000010:2.1:1787760855.689207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8593800.
00000080:00000010:0.1:1787760855.762199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060380.
00000080:00000010:0.1:1787760855.762229:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060920.
00010000:00000010:2.1:1787760856.259424:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2700.
00010000:00000010:2.1:1787760856.259450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8760000.
00000080:00000010:2.1:1787760856.333192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060440.
00000080:00000010:2.1:1787760856.333207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c61aeaa0.
00010000:00000010:0.1:1787760856.493323:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784ea00.
00010000:00000010:0.1:1787760856.493346:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa52200.
00000080:00000010:2.1:1787760856.527575:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c61ae3c0.
00000080:00000010:2.1:1787760856.527594:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0ecc440.
00010000:00000010:0.1:1787760856.861615:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46900.
00010000:00000010:0.1:1787760856.861631:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8592600.
00000080:00000010:0.1:1787760856.876259:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c61ae3c0.
00000080:00000010:0.1:1787760856.876277:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060920.
00010000:00000010:3.1:1787760857.071189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46e00.
00010000:00000010:3.1:1787760857.071200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8593600.
00000080:00000010:2.1:1787760857.083210:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d880.
00000080:00000010:2.1:1787760857.083223:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060780.
00010000:00000010:3.1:1787760857.304200:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46900.
00010000:00000010:3.1:1787760857.304209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8593600.
00000080:00000010:0.1:1787760857.318185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811de00.
00000080:00000010:0.1:1787760857.318201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060620.
00010000:00000010:1.1:1787760857.530193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2400.
00010000:00000010:1.1:1787760857.530208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8761e00.
00000080:00000010:1.1:1787760857.545176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0ecc7e0.
00000080:00000010:1.1:1787760857.545187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c61aeaa0.
00010000:00000010:1.1:1787760857.700255:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784ea00.
00010000:00000010:1.1:1787760857.700263:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa52c00.
00000080:00000010:0.1:1787760857.726175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0ecc440.
00000080:00000010:0.1:1787760857.726188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0ecc7e0.
00010000:00000010:2.1:1787760857.997186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784ea00.
00010000:00000010:2.1:1787760857.997203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8760a00.
00000080:00000010:2.1:1787760858.046256:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060240.
00000080:00000010:2.1:1787760858.046268:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ecc80600.
00010000:00000010:3.1:1787760858.386606:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2d00.
00010000:00000010:3.1:1787760858.386618:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8760200.
00000080:00000010:3.1:1787760858.422677:0:257571:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811de20.
00000080:00000010:3.1:1787760858.422698:0:257571:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ecc80820.
00010000:00000010:2.1:1787760858.790053:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784ef00.
00010000:00000010:2.1:1787760858.790066:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53200.
00000080:00000010:1.1:1787760858.811543:0:1365:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060920.
00000080:00000010:1.1:1787760858.811557:0:1365:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0ecc440.
00010000:00000010:0.1:1787760859.022008:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30100.
00010000:00000010:0.1:1787760859.022024:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79abc00.
00000080:00000010:0.1:1787760859.044172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ecc80dc0.
00000080:00000010:0.1:1787760859.044185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811da60.
00010000:00000010:1.1:1787760859.303542:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30b00.
00010000:00000010:1.1:1787760859.303557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aac00.
00000080:00000010:1.1:1787760859.330546:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0ecc440.
00000080:00000010:1.1:1787760859.330561:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d3e0.
00010000:00000010:2.1:1787760859.462190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2f00.
00010000:00000010:2.1:1787760859.462205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8592c00.
00000080:00000010:0.1:1787760859.489252:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060920.
00000080:00000010:0.1:1787760859.489265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060620.
00010000:00000010:1.1:1787760859.664754:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2f00.
00010000:00000010:1.1:1787760859.664765:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8592c00.
00000080:00000010:1.1:1787760859.677351:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca140.
00000080:00000010:1.1:1787760859.677365:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060620.
00010000:00000010:2.1:1787760859.866270:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2400.
00010000:00000010:2.1:1787760859.866280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8761400.
00000080:00000010:3.1:1787760859.883430:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ecc80080.
00000080:00000010:3.1:1787760859.883442:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ecc80a20.
00010000:00000010:2.1:1787760860.114187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2300.
00010000:00000010:2.1:1787760860.114203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8760e00.
00000080:00000010:0.1:1787760860.137201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060620.
00000080:00000010:0.1:1787760860.137219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ecc80080.
00010000:00000010:2.1:1787760860.433174:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2400.
00010000:00000010:2.1:1787760860.433184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8761000.
00000080:00000010:0.1:1787760860.466837:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060920.
00000080:00000010:0.1:1787760860.466852:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ecc80080.
00010000:00000010:1.1:1787760860.775754:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2f00.
00010000:00000010:1.1:1787760860.775769:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8760800.
00000080:00000010:1.1:1787760860.834258:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca400.
00000080:00000010:1.1:1787760860.834278:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ecc80080.
00010000:00000010:2.1:1787760861.271617:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784ea00.
00010000:00000010:2.1:1787760861.271632:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53200.
00000080:00000010:2.1:1787760861.296483:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060920.
00000080:00000010:2.1:1787760861.296503:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca520.
00010000:00000010:0.1:1787760861.695219:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150000.
00010000:00000010:0.1:1787760861.695235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53200.
00000080:00000010:0.1:1787760861.734554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ecc80080.
00000080:00000010:0.1:1787760861.734634:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca400.
00010000:00000010:2.1:1787760861.906340:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46900.
00010000:00000010:2.1:1787760861.906354:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8593400.
00000080:00000010:3.1:1787760861.958222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060e60.
00000080:00000010:3.1:1787760861.958247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060920.
00010000:00000010:1.1:1787760862.203373:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71700.
00010000:00000010:1.1:1787760862.203389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8760e00.
00000080:00000010:2.1:1787760862.277565:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cad80.
00000080:00000010:2.1:1787760862.277582:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ecc80080.
00010000:00000010:2.1:1787760862.601920:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71600.
00010000:00000010:2.1:1787760862.601933:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8761400.
00000080:00000010:2.1:1787760862.631195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060e60.
00000080:00000010:2.1:1787760862.631212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ecc80080.
00010000:00000010:3.1:1787760863.238188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71800.
00010000:00000010:3.1:1787760863.238204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8760a00.
00000080:00000010:0.1:1787760863.277180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811daa0.
00000080:00000010:0.1:1787760863.277191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750f9df7a20.
00010000:00000010:0.1:1787760863.533245:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150a00.
00010000:00000010:0.1:1787760863.533258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53200.
00000080:00000010:0.1:1787760863.567216:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750c7e60b20.
00000080:00000010:0.1:1787760863.567235:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca980.
00010000:00000010:2.1:1787760863.978336:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71c00.
00010000:00000010:2.1:1787760863.978350:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8760600.
00000080:00000010:0.1:1787760864.059349:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d960.
00000080:00000010:0.1:1787760864.059373:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750c7e60b20.
00010000:00000010:3.1:1787760864.473178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71f00.
00010000:00000010:3.1:1787760864.473194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8592600.
00000080:00000010:3.1:1787760864.488180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d560.
00000080:00000010:3.1:1787760864.488210:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751ca060e60.
00010000:00000010:3.1:1787760864.839218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30400.
00010000:00000010:3.1:1787760864.839237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8761400.
00000080:00000010:0.1:1787760864.880318:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d560.
00000080:00000010:0.1:1787760864.880341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750c7e60640.
00010000:00000010:1.1:1787760865.283127:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150300.
00010000:00000010:1.1:1787760865.283176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53200.
00000080:00000010:1.1:1787760865.306164:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cafe0.
00000080:00000010:1.1:1787760865.306190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cab00.
00010000:00000010:1.1:1787760865.774163:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71d00.
00010000:00000010:1.1:1787760865.774170:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aaa00.
00000080:00000010:1.1:1787760865.810736:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cab00.
00000080:00000010:1.1:1787760865.810751:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811de60.
00010000:00000010:0.1:1787760866.202201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150800.
00010000:00000010:0.1:1787760866.202223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53200.
00000080:00000010:3.1:1787760866.279355:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750c7e60f40.
00000080:00000010:3.1:1787760866.279375:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cab00.
00010000:00000010:3.1:1787760866.581332:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30f00.
00010000:00000010:3.1:1787760866.581348:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79ab800.
00000080:00000010:3.1:1787760866.600393:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811da20.
00000080:00000010:3.1:1787760866.630207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811db40.
00010000:00000010:1.1:1787760867.015323:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150b00.
00010000:00000010:1.1:1787760867.015340:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53200.
00000080:00000010:1.1:1787760867.117316:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cafe0.
00000080:00000010:1.1:1787760867.117346:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750c7e60380.
00010000:00000010:2.1:1787760867.443221:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71100.
00010000:00000010:2.1:1787760867.443248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8782800.
00000080:00000010:2.1:1787760867.459484:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d01b5d40.
00000080:00000010:2.1:1787760867.459504:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750c7e60f40.
00010000:00000010:1.1:1787760867.682188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71200.
00010000:00000010:1.1:1787760867.682284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aaa00.
00000080:00000010:1.1:1787760867.721579:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cafe0.
00000080:00000010:1.1:1787760867.721604:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811dbc0.
00010000:00000010:2.1:1787760868.208202:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150c00.
00010000:00000010:2.1:1787760868.208226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85db800.
00000080:00000010:0.1:1787760868.263249:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d01b5d40.
00000080:00000010:0.1:1787760868.263267:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cafe0.
00010000:00000010:0.1:1787760868.626278:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71200.
00010000:00000010:0.1:1787760868.626293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8783200.
00000080:00000010:0.1:1787760868.652199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750f9df7a20.
00000080:00000010:0.1:1787760868.652216:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d960.
00010000:00000010:1.1:1787760869.331729:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71200.
00010000:00000010:1.1:1787760869.331745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8783200.
00000080:00000010:1.1:1787760869.357527:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca300.
00000080:00000010:1.1:1787760869.357540:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750f9df7a20.
00010000:00000010:0.1:1787760869.868177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71d00.
00010000:00000010:0.1:1787760869.868193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8783600.
00000080:00000010:0.1:1787760869.891170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6ab60.
00000080:00000010:0.1:1787760869.891190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d960.
00010000:00000010:2.1:1787760870.273060:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71f00.
00010000:00000010:2.1:1787760870.273076:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8783600.
00000080:00000010:0.1:1787760870.332670:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219bc0.
00000080:00000010:0.1:1787760870.332684:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6ab60.
00010000:00000010:3.1:1787760870.752439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71800.
00010000:00000010:3.1:1787760870.752452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8783c00.
00000080:00000010:3.1:1787760870.828200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d280.
00000080:00000010:3.1:1787760870.828221:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6ab60.
00010000:00000010:1.1:1787760871.397174:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71700.
00010000:00000010:1.1:1787760871.397192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8783e00.
00000080:00000010:1.1:1787760871.436358:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca300.
00000080:00000010:1.1:1787760871.436370:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a740.
00010000:00000010:2.1:1787760871.790533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30b00.
00010000:00000010:2.1:1787760871.790547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79abc00.
00000080:00000010:1.1:1787760871.840250:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219900.
00000080:00000010:1.1:1787760871.840268:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d280.
00010000:00000010:0.1:1787760872.247812:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71700.
00010000:00000010:0.1:1787760872.247822:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85dba00.
00000080:00000010:0.1:1787760872.253004:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a780.
00000080:00000010:0.1:1787760872.253018:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cadc0.
00010000:00000010:2.1:1787760872.535068:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71800.
00010000:00000010:2.1:1787760872.535079:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8782c00.
00000080:00000010:0.1:1787760872.604765:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219820.
00000080:00000010:0.1:1787760872.604783:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a780.
00010000:00000010:2.1:1787760873.042297:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71f00.
00010000:00000010:2.1:1787760873.042305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aac00.
00000080:00000010:2.1:1787760873.077401:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219820.
00000080:00000010:2.1:1787760873.077415:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d5a0.
00010000:00000010:0.1:1787760873.636494:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84eb700.
00010000:00000010:0.1:1787760873.636509:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8593600.
00000080:00000010:0.1:1787760873.652230:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a0e0.
00000080:00000010:0.1:1787760873.652243:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219820.
00010000:00000010:3.1:1787760874.107950:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84eb600.
00010000:00000010:3.1:1787760874.107965:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa57a00.
00000080:00000010:3.1:1787760874.124524:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d540.
00000080:00000010:3.1:1787760874.124542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c62194c0.
00010000:00000010:0.1:1787760874.671241:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150900.
00010000:00000010:0.1:1787760874.671258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85da800.
00000080:00000010:0.1:1787760874.699515:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a0e0.
00000080:00000010:0.1:1787760874.699531:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca900.
00010000:00000010:3.1:1787760875.120199:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71800.
00010000:00000010:3.1:1787760875.120216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8783800.
00000080:00000010:3.1:1787760875.170600:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d540.
00000080:00000010:3.1:1787760875.170618:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a0e0.
00010000:00000010:2.1:1787760875.513217:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84eb300.
00010000:00000010:2.1:1787760875.513232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aba00.
00000080:00000010:0.1:1787760875.569968:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219e40.
00000080:00000010:0.1:1787760875.569989:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d540.
00010000:00000010:2.1:1787760875.853600:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1e29e00.
00010000:00000010:2.1:1787760875.853616:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aae00.
00000080:00000010:2.1:1787760875.883379:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219900.
00000080:00000010:2.1:1787760875.883397:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811dde0.
00010000:00000010:3.1:1787760876.164299:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84eb800.
00010000:00000010:3.1:1787760876.164315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa57e00.
00000080:00000010:0.1:1787760876.191287:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d740.
00000080:00000010:0.1:1787760876.191299:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219900.
00010000:00000010:2.1:1787760876.470343:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150400.
00010000:00000010:2.1:1787760876.470354:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85db200.
00000080:00000010:2.1:1787760876.484176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219a00.
00000080:00000010:2.1:1787760876.484189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca740.
00010000:00000010:2.1:1787760876.773176:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467a900.
00010000:00000010:2.1:1787760876.773210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa57200.
00000080:00000010:3.1:1787760876.787456:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219820.
00000080:00000010:3.1:1787760876.787470:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c62190e0.
00010000:00000010:0.1:1787760877.157185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84ebf00.
00010000:00000010:0.1:1787760877.157213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa56600.
00000080:00000010:1.1:1787760877.191372:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219820.
00000080:00000010:1.1:1787760877.191393:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c62190e0.
00010000:00000010:2.1:1787760877.339183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150200.
00010000:00000010:2.1:1787760877.339193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85dbc00.
00000080:00000010:2.1:1787760877.364276:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c62198c0.
00000080:00000010:2.1:1787760877.364288:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca7e0.
00010000:00000010:2.1:1787760877.639703:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84ebd00.
00010000:00000010:2.1:1787760877.639715:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa57000.
00000080:00000010:2.1:1787760877.693438:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219900.
00000080:00000010:2.1:1787760877.693459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219fa0.
00010000:00000010:2.1:1787760877.866881:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84ebd00.
00010000:00000010:2.1:1787760877.866897:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa57000.
00000080:00000010:3.1:1787760877.884173:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219900.
00000080:00000010:3.1:1787760877.884187:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219fa0.
00010000:00000010:0.1:1787760878.073349:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84ebd00.
00010000:00000010:0.1:1787760878.073366:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aba00.
00000080:00000010:0.1:1787760878.091192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a700.
00000080:00000010:0.1:1787760878.091282:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d6e0.
00010000:00000010:0.1:1787760878.377346:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467a100.
00010000:00000010:0.1:1787760878.377360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa57c00.
00000080:00000010:0.1:1787760878.392169:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a700.
00000080:00000010:2.1:1787760878.414416:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219e40.
00010000:00000010:2.1:1787760878.705462:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467a100.
00010000:00000010:2.1:1787760878.705477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aaa00.
00000080:00000010:3.1:1787760878.737544:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219220.
00000080:00000010:3.1:1787760878.737564:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811dac0.
00010000:00000010:1.1:1787760879.109187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150400.
00010000:00000010:1.1:1787760879.109202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85dbe00.
00000080:00000010:1.1:1787760879.139358:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca200.
00000080:00000010:1.1:1787760879.139370:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca6c0.
00010000:00000010:0.1:1787760879.586216:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150400.
00010000:00000010:0.1:1787760879.586224:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85dbe00.
00000080:00000010:0.1:1787760879.627807:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a680.
00000080:00000010:0.1:1787760879.627824:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a980.
00010000:00000010:0.1:1787760879.951404:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467a700.
00010000:00000010:0.1:1787760879.951418:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa57e00.
00000080:00000010:0.1:1787760879.973460:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a980.
00000080:00000010:0.1:1787760879.973486:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c62190e0.
00010000:00000010:2.1:1787760880.330300:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150f00.
00010000:00000010:2.1:1787760880.330322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85daa00.
00000080:00000010:2.1:1787760880.354181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c62191a0.
00000080:00000010:2.1:1787760880.354194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca200.
00010000:00000010:1.1:1787760880.672361:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467a700.
00010000:00000010:1.1:1787760880.672379:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8782c00.
00000080:00000010:1.1:1787760880.687184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca4c0.
00000080:00000010:1.1:1787760880.719722:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219e40.
00010000:00000010:2.1:1787760881.009901:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467a600.
00010000:00000010:2.1:1787760881.009918:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8783e00.
00000080:00000010:0.1:1787760881.053190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c62199a0.
00000080:00000010:0.1:1787760881.053207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a980.
00010000:00000010:0.1:1787760881.301184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150800.
00010000:00000010:0.1:1787760881.301201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85daa00.
00000080:00000010:0.1:1787760881.313462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a980.
00000080:00000010:0.1:1787760881.321170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cac80.
00010000:00000010:2.1:1787760881.624277:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1e29800.
00010000:00000010:2.1:1787760881.624292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa57800.
00000080:00000010:0.1:1787760881.672835:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219fa0.
00000080:00000010:0.1:1787760881.672852:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c62190e0.
00010000:00000010:0.1:1787760882.005181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467a800.
00010000:00000010:0.1:1787760882.005188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8783600.
00000080:00000010:0.1:1787760882.029832:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a140.
00000080:00000010:0.1:1787760882.029846:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a680.
00010000:00000010:1.1:1787760882.195420:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150a00.
00010000:00000010:1.1:1787760882.195436:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85dbe00.
00000080:00000010:1.1:1787760882.207188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cad20.
00000080:00000010:1.1:1787760882.207202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cae00.
00010000:00000010:2.1:1787760882.433182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467ac00.
00010000:00000010:2.1:1787760882.433196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8782800.
00000080:00000010:2.1:1787760882.481676:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a740.
00000080:00000010:2.1:1787760882.481698:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a140.
00010000:00000010:3.1:1787760882.800184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84eb000.
00010000:00000010:3.1:1787760882.800196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa56600.
00000080:00000010:2.1:1787760882.832522:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d100.
00000080:00000010:2.1:1787760882.832543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219e40.
00010000:00000010:1.1:1787760883.161439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467ae00.
00010000:00000010:1.1:1787760883.161453:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aaa00.
00000080:00000010:1.1:1787760883.186188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cae00.
00000080:00000010:1.1:1787760883.186208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d4e0.
00010000:00000010:0.1:1787760883.444239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84ebd00.
00010000:00000010:0.1:1787760883.444255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa56200.
00000080:00000010:0.1:1787760883.463421:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6ab60.
00000080:00000010:0.1:1787760883.463436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219820.
00010000:00000010:3.1:1787760883.782365:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1e29800.
00010000:00000010:3.1:1787760883.782379:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aba00.
00000080:00000010:1.1:1787760883.832183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d9c0.
00000080:00000010:1.1:1787760883.832196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d380.
00010000:00000010:2.1:1787760884.148191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467af00.
00010000:00000010:2.1:1787760884.148206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa56200.
00000080:00000010:0.1:1787760884.186272:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c62194e0.
00000080:00000010:0.1:1787760884.186298:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6219220.
00010000:00000010:2.1:1787760884.448184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150800.
00010000:00000010:2.1:1787760884.448476:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85db600.
00000080:00000010:3.1:1787760884.470493:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c62194e0.
00000080:00000010:3.1:1787760884.470513:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca800.
00010000:00000010:0.1:1787760884.770669:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84ebc00.
00010000:00000010:0.1:1787760884.770685:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa56e00.
00000080:00000010:0.1:1787760884.790184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a740.
00000080:00000010:0.1:1787760884.790199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c62194c0.
00010000:00000010:0.1:1787760885.175541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84eb600.
00010000:00000010:0.1:1787760885.175557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa57600.
00000080:00000010:0.1:1787760885.193682:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a740.
00000080:00000010:0.1:1787760885.193696:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c62190e0.
00010000:00000010:2.1:1787760885.500176:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467a500.
00010000:00000010:2.1:1787760885.500194:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8782800.
00000080:00000010:2.1:1787760885.547187:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c62194e0.
00000080:00000010:2.1:1787760885.547197:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a740.
00010000:00000010:3.1:1787760885.820942:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84eb600.
00010000:00000010:3.1:1787760885.820959:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa56000.
00000080:00000010:0.1:1787760885.844472:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811dd00.
00000080:00000010:0.1:1787760885.844485:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c62194e0.
00010000:00000010:3.1:1787760886.184179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84ebd00.
00010000:00000010:3.1:1787760886.184205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa56e00.
00000080:00000010:0.1:1787760886.239225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d840.
00000080:00000010:0.1:1787760886.239242:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c62190e0.
00010000:00000010:3.1:1787760886.529179:0:257571:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84ebd00.
00010000:00000010:3.1:1787760886.529191:0:257571:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8622800.
00000080:00000010:0.1:1787760886.550172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d500.
00000080:00000010:0.1:1787760886.550200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a140.
00010000:00000010:0.1:1787760886.832918:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467af00.
00010000:00000010:0.1:1787760886.832933:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8782800.
00000080:00000010:0.1:1787760886.850177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d960.
00000080:00000010:0.1:1787760886.850190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a140.
00010000:00000010:2.1:1787760887.167471:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84eb600.
00010000:00000010:2.1:1787760887.167493:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8622a00.
00000080:00000010:0.1:1787760887.210814:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0768840.
00000080:00000010:0.1:1787760887.210836:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303b60.
00010000:00000010:2.1:1787760887.562191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84eb600.
00010000:00000010:2.1:1787760887.562207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8782800.
00000080:00000010:0.1:1787760887.578170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0768000.
00000080:00000010:0.1:1787760887.578182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f5d1d960.
00010000:00000010:1.1:1787760887.796369:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467ae00.
00010000:00000010:1.1:1787760887.796385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8622200.
00000080:00000010:0.1:1787760887.837337:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca0a0.
00000080:00000010:0.1:1787760887.837354:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0768d60.
00010000:00000010:0.1:1787760888.123254:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467a600.
00010000:00000010:0.1:1787760888.123269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870e200.
00000080:00000010:0.1:1787760888.164560:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a140.
00000080:00000010:0.1:1787760888.164580:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303b60.
00010000:00000010:2.1:1787760888.502570:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150800.
00010000:00000010:2.1:1787760888.502586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85db800.
00000080:00000010:0.1:1787760888.544394:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0768b20.
00000080:00000010:0.1:1787760888.544515:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca5a0.
00010000:00000010:3.1:1787760888.874332:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150800.
00010000:00000010:3.1:1787760888.874360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85db800.
00000080:00000010:2.1:1787760888.893521:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d220.
00000080:00000010:2.1:1787760888.893544:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cade0.
00010000:00000010:2.1:1787760889.305254:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467a100.
00010000:00000010:2.1:1787760889.305266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8623400.
00000080:00000010:0.1:1787760889.326176:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0768d60.
00000080:00000010:0.1:1787760889.326196:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0768b20.
00010000:00000010:2.1:1787760889.699366:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84ebd00.
00010000:00000010:2.1:1787760889.699380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8622e00.
00000080:00000010:2.1:1787760889.759173:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0768840.
00000080:00000010:2.1:1787760889.759193:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0768000.
00010000:00000010:2.1:1787760890.112275:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467ae00.
00010000:00000010:2.1:1787760890.112291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870f400.
00000080:00000010:2.1:1787760890.159681:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0768840.
00000080:00000010:2.1:1787760890.159694:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf840.
00010000:00000010:0.1:1787760890.478297:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467ae00.
00010000:00000010:0.1:1787760890.478312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8622200.
00000080:00000010:2.1:1787760890.545969:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf0e0.
00000080:00000010:2.1:1787760890.545989:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0768d60.
00010000:00000010:2.1:1787760890.800677:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150800.
00010000:00000010:2.1:1787760890.800695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85db600.
00000080:00000010:2.1:1787760890.845331:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0768d60.
00000080:00000010:2.1:1787760890.845350:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cacc0.
00010000:00000010:2.1:1787760891.158359:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039000.
00010000:00000010:2.1:1787760891.158369:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870ee00.
00000080:00000010:0.1:1787760891.175273:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0768000.
00000080:00000010:0.1:1787760891.175290:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acff20.
00010000:00000010:2.1:1787760891.549305:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039700.
00010000:00000010:2.1:1787760891.549321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8623600.
00000080:00000010:0.1:1787760891.563529:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0768d60.
00000080:00000010:0.1:1787760891.563550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0768b20.
00010000:00000010:1.1:1787760892.079265:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518d500.
00010000:00000010:1.1:1787760892.079286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8623200.
00000080:00000010:1.1:1787760892.122949:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca6a0.
00000080:00000010:1.1:1787760892.122967:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0768d60.
00010000:00000010:1.1:1787760892.432489:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873d300.
00010000:00000010:1.1:1787760892.432507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870f000.
00000080:00000010:1.1:1787760892.450595:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca6a0.
00000080:00000010:1.1:1787760892.450618:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf2c0.
00010000:00000010:0.1:1787760892.810196:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039300.
00010000:00000010:0.1:1787760892.810214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870e800.
00000080:00000010:1.1:1787760892.903789:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf860.
00000080:00000010:1.1:1787760892.903812:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfe80.
00010000:00000010:2.1:1787760893.306619:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039800.
00010000:00000010:2.1:1787760893.306637:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870e800.
00000080:00000010:0.1:1787760893.343327:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751cfd29bc0.
00000080:00000010:0.1:1787760893.343341:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf540.
00010000:00000010:2.1:1787760893.684774:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039c00.
00010000:00000010:2.1:1787760893.684789:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870f200.
00000080:00000010:0.1:1787760893.730384:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751cfd29660.
00000080:00000010:0.1:1787760893.730506:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acffe0.
00010000:00000010:2.1:1787760894.008202:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039f00.
00010000:00000010:2.1:1787760894.008228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8623400.
00000080:00000010:0.1:1787760894.062362:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751cfd29e20.
00000080:00000010:0.1:1787760894.062379:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751cfd29200.
00010000:00000010:1.1:1787760894.445286:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039d00.
00010000:00000010:1.1:1787760894.445301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870fc00.
00000080:00000010:1.1:1787760894.462971:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca6a0.
00000080:00000010:1.1:1787760894.462986:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfb20.
00010000:00000010:1.1:1787760894.833302:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039400.
00010000:00000010:1.1:1787760894.833318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870fe00.
00000080:00000010:1.1:1787760894.857995:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca6a0.
00000080:00000010:1.1:1787760894.858014:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf140.
00010000:00000010:1.1:1787760895.277187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039100.
00010000:00000010:1.1:1787760895.277202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3c200.
00000080:00000010:1.1:1787760895.341469:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca6a0.
00000080:00000010:1.1:1787760895.341484:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d9c0.
00010000:00000010:0.1:1787760895.671180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635c00.
00010000:00000010:0.1:1787760895.671263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85db800.
00000080:00000010:0.1:1787760895.725322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca2e0.
00000080:00000010:0.1:1787760895.725338:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca3a0.
00010000:00000010:1.1:1787760896.105424:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039400.
00010000:00000010:1.1:1787760896.105438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870ea00.
00000080:00000010:1.1:1787760896.171521:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca8a0.
00000080:00000010:1.1:1787760896.171531:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf620.
00010000:00000010:3.1:1787760896.510358:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039f00.
00010000:00000010:3.1:1787760896.510371:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870ea00.
00000080:00000010:3.1:1787760896.544541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d4e0.
00000080:00000010:3.1:1787760896.544558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf7c0.
00010000:00000010:2.1:1787760897.066185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635f00.
00010000:00000010:2.1:1787760897.066200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981000.
00000080:00000010:0.1:1787760897.106184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751cfd290e0.
00000080:00000010:0.1:1787760897.106205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caca0.
00010000:00000010:0.1:1787760897.607187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635500.
00010000:00000010:0.1:1787760897.607207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870fe00.
00000080:00000010:0.1:1787760897.653121:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf9e0.
00000080:00000010:0.1:1787760897.653147:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfce0.
00010000:00000010:2.1:1787760897.950181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635900.
00010000:00000010:2.1:1787760897.950196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980800.
00000080:00000010:0.1:1787760898.019173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751cfd29660.
00000080:00000010:0.1:1787760898.019198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca820.
00010000:00000010:3.1:1787760898.372178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039700.
00010000:00000010:3.1:1787760898.372194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8623000.
00000080:00000010:3.1:1787760898.428718:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d4e0.
00000080:00000010:3.1:1787760898.428746:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751cfd29200.
00010000:00000010:0.1:1787760898.753915:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039700.
00010000:00000010:0.1:1787760898.753927:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8623a00.
00000080:00000010:1.1:1787760898.790277:0:19:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf520.
00000080:00000010:1.1:1787760898.790289:0:19:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751cfd291e0.
00010000:00000010:0.1:1787760899.187264:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635100.
00010000:00000010:0.1:1787760899.187279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981200.
00000080:00000010:0.1:1787760899.218561:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfb80.
00000080:00000010:0.1:1787760899.218579:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca640.
00010000:00000010:0.1:1787760899.641204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518df00.
00010000:00000010:0.1:1787760899.641224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3d400.
00000080:00000010:0.1:1787760899.686580:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfb80.
00000080:00000010:0.1:1787760899.694305:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d6e0.
00010000:00000010:2.1:1787760899.925702:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518dd00.
00010000:00000010:2.1:1787760899.925719:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3ce00.
00000080:00000010:0.1:1787760899.980176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751cfd29660.
00000080:00000010:0.1:1787760899.980193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d740.
00010000:00000010:2.1:1787760900.280855:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039700.
00010000:00000010:2.1:1787760900.280871:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8623a00.
00000080:00000010:0.1:1787760900.307810:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751cfd291e0.
00000080:00000010:0.1:1787760900.307829:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751cfd290e0.
00010000:00000010:1.1:1787760900.689093:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039000.
00010000:00000010:1.1:1787760900.689111:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8623a00.
00000080:00000010:1.1:1787760900.710454:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca5c0.
00000080:00000010:1.1:1787760900.710464:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751cfd29bc0.
00010000:00000010:3.1:1787760901.029850:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039400.
00010000:00000010:3.1:1787760901.029869:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870f400.
00000080:00000010:3.1:1787760901.060841:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811dde0.
00000080:00000010:3.1:1787760901.060857:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf440.
00010000:00000010:0.1:1787760901.590382:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635000.
00010000:00000010:0.1:1787760901.590394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981600.
00000080:00000010:0.1:1787760901.625582:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acff80.
00000080:00000010:0.1:1787760901.625601:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca5c0.
00010000:00000010:3.1:1787760902.213759:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635700.
00010000:00000010:3.1:1787760902.213775:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870f400.
00000080:00000010:3.1:1787760902.213780:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811dde0.
00000080:00000010:3.1:1787760902.213789:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acff80.
00010000:00000010:2.1:1787760902.668188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635600.
00010000:00000010:2.1:1787760902.668212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981c00.
00000080:00000010:3.1:1787760902.688168:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751cfd29200.
00000080:00000010:3.1:1787760902.688182:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caaa0.
00010000:00000010:2.1:1787760903.294192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039700.
00010000:00000010:2.1:1787760903.294206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870f400.
00000080:00000010:0.1:1787760903.322314:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf5c0.
00000080:00000010:0.1:1787760903.322339:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf760.
00010000:00000010:2.1:1787760903.705175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635800.
00010000:00000010:2.1:1787760903.705448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981e00.
00000080:00000010:0.1:1787760903.760671:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751cfd29200.
00000080:00000010:0.1:1787760903.760685:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cafa0.
00010000:00000010:1.1:1787760904.093930:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635800.
00010000:00000010:1.1:1787760904.093957:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479f400.
00000080:00000010:1.1:1787760904.111985:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cad00.
00000080:00000010:1.1:1787760904.111998:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfa80.
00010000:00000010:3.1:1787760904.457245:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873de00.
00010000:00000010:3.1:1787760904.457261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3da00.
00000080:00000010:3.1:1787760904.493845:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811dde0.
00000080:00000010:3.1:1787760904.493861:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751cfd29bc0.
00010000:00000010:2.1:1787760904.882031:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635800.
00010000:00000010:2.1:1787760904.882046:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980c00.
00000080:00000010:3.1:1787760904.905879:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65760.
00000080:00000010:3.1:1787760904.905899:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cad00.
00010000:00000010:3.1:1787760905.391838:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635700.
00010000:00000010:3.1:1787760905.391869:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980a00.
00000080:00000010:3.1:1787760905.429453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d540.
00000080:00000010:3.1:1787760905.429471:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca0e0.
00010000:00000010:1.1:1787760905.938789:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748d00.
00010000:00000010:1.1:1787760905.938806:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479e800.
00000080:00000010:3.1:1787760906.001228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca2c0.
00000080:00000010:3.1:1787760906.001244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf2a0.
00010000:00000010:2.1:1787760906.642849:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635000.
00010000:00000010:2.1:1787760906.642865:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479e600.
00000080:00000010:3.1:1787760906.730588:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65ec0.
00000080:00000010:3.1:1787760906.730613:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfd20.
00010000:00000010:3.1:1787760907.178212:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748100.
00010000:00000010:3.1:1787760907.178244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479f600.
00000080:00000010:3.1:1787760907.258310:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65280.
00000080:00000010:3.1:1787760907.258325:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfc60.
00010000:00000010:2.1:1787760907.819093:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748200.
00010000:00000010:2.1:1787760907.819111:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8622600.
00000080:00000010:0.1:1787760907.883213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65980.
00000080:00000010:0.1:1787760907.883226:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d280.
00010000:00000010:1.1:1787760908.299225:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635100.
00010000:00000010:1.1:1787760908.299240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980200.
00000080:00000010:1.1:1787760908.366211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cad60.
00000080:00000010:1.1:1787760908.366228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caf20.
00010000:00000010:0.1:1787760908.807219:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748a00.
00010000:00000010:0.1:1787760908.807234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479fc00.
00000080:00000010:1.1:1787760908.842203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf6e0.
00000080:00000010:1.1:1787760908.842217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf7a0.
00010000:00000010:3.1:1787760909.299178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635100.
00010000:00000010:3.1:1787760909.299194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980200.
00000080:00000010:3.1:1787760909.319249:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811dbc0.
00000080:00000010:3.1:1787760909.319272:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caf20.
00010000:00000010:1.1:1787760909.686885:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873df00.
00010000:00000010:1.1:1787760909.686899:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3c800.
00000080:00000010:1.1:1787760909.729265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cad60.
00000080:00000010:1.1:1787760909.729284:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811de60.
00010000:00000010:2.1:1787760910.054362:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748300.
00010000:00000010:2.1:1787760910.054378:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479ec00.
00000080:00000010:0.1:1787760910.079201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65640.
00000080:00000010:0.1:1787760910.079216:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf080.
00010000:00000010:3.1:1787760910.549958:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635500.
00010000:00000010:3.1:1787760910.549973:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980400.
00000080:00000010:3.1:1787760910.625347:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d560.
00000080:00000010:3.1:1787760910.625358:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca580.
00010000:00000010:2.1:1787760910.978206:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748b00.
00010000:00000010:2.1:1787760910.978222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479e200.
00000080:00000010:0.1:1787760911.023197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65140.
00000080:00000010:0.1:1787760911.023217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf300.
00010000:00000010:0.1:1787760911.636360:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748c00.
00010000:00000010:0.1:1787760911.636375:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8763600.
00000080:00000010:0.1:1787760911.677173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf300.
00000080:00000010:0.1:1787760911.677185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65b80.
00010000:00000010:3.1:1787760912.266312:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748c00.
00010000:00000010:3.1:1787760912.266328:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479f800.
00000080:00000010:2.1:1787760912.327384:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d560.
00000080:00000010:2.1:1787760912.327407:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf300.
00010000:00000010:2.1:1787760912.900193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635500.
00010000:00000010:2.1:1787760912.900216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980400.
00000080:00000010:0.1:1787760912.957417:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65b40.
00000080:00000010:0.1:1787760912.957426:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cae60.
00010000:00000010:1.1:1787760913.655171:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635000.
00010000:00000010:1.1:1787760913.655184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980a00.
00000080:00000010:1.1:1787760913.709813:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca2a0.
00000080:00000010:1.1:1787760913.709828:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca660.
00010000:00000010:0.1:1787760914.037181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635000.
00010000:00000010:0.1:1787760914.037193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980a00.
00000080:00000010:0.1:1787760914.095544:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca2a0.
00000080:00000010:0.1:1787760914.095563:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca660.
00010000:00000010:2.1:1787760914.303331:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748b00.
00010000:00000010:2.1:1787760914.303345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479f800.
00000080:00000010:3.1:1787760914.322259:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65680.
00000080:00000010:3.1:1787760914.322274:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfa40.
00010000:00000010:3.1:1787760914.668302:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748300.
00010000:00000010:3.1:1787760914.668316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8762600.
00000080:00000010:3.1:1787760914.739227:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811df00.
00000080:00000010:3.1:1787760914.739244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65ba0.
00010000:00000010:0.1:1787760915.141344:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748100.
00010000:00000010:0.1:1787760915.141362:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479ec00.
00000080:00000010:0.1:1787760915.181172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfd80.
00000080:00000010:0.1:1787760915.181185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfd60.
00010000:00000010:2.1:1787760915.485451:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748d00.
00010000:00000010:2.1:1787760915.485466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479fe00.
00000080:00000010:0.1:1787760915.499257:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b654a0.
00000080:00000010:0.1:1787760915.499272:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfd60.
00010000:00000010:0.1:1787760915.744178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873d400.
00010000:00000010:0.1:1787760915.744193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981e00.
00000080:00000010:0.1:1787760915.811183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfd60.
00000080:00000010:0.1:1787760915.811197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca180.
00010000:00000010:0.1:1787760916.014627:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635000.
00010000:00000010:0.1:1787760916.014642:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981600.
00000080:00000010:0.1:1787760916.037179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfd60.
00000080:00000010:0.1:1787760916.037195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caa80.
00010000:00000010:2.1:1787760916.392364:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635000.
00010000:00000010:2.1:1787760916.392378:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981200.
00000080:00000010:3.1:1787760916.452291:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65380.
00000080:00000010:3.1:1787760916.452305:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca6e0.
00010000:00000010:0.1:1787760917.106222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518df00.
00010000:00000010:0.1:1787760917.106237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980800.
00000080:00000010:0.1:1787760917.121245:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfd60.
00000080:00000010:0.1:1787760917.121257:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca7a0.
00010000:00000010:3.1:1787760917.705899:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748300.
00010000:00000010:3.1:1787760917.705915:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8762600.
00000080:00000010:3.1:1787760917.778168:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811df00.
00000080:00000010:3.1:1787760917.778180:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65d00.
00010000:00000010:2.1:1787760918.523169:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635000.
00010000:00000010:2.1:1787760918.523187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981000.
00000080:00000010:0.1:1787760918.582270:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b651e0.
00000080:00000010:0.1:1787760918.582285:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cabe0.
00010000:00000010:2.1:1787760919.214705:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786f100.
00010000:00000010:2.1:1787760919.214720:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981000.
00000080:00000010:0.1:1787760919.269194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65720.
00000080:00000010:0.1:1787760919.269215:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca160.
00010000:00000010:0.1:1787760919.652703:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748300.
00010000:00000010:0.1:1787760919.652718:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981200.
00000080:00000010:0.1:1787760919.697171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf740.
00000080:00000010:0.1:1787760919.697184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca720.
00010000:00000010:1.1:1787760920.278737:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466c00.
00010000:00000010:1.1:1787760920.278751:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8763200.
00000080:00000010:1.1:1787760920.326621:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca8e0.
00000080:00000010:1.1:1787760920.326636:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65c00.
00010000:00000010:1.1:1787760920.988455:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466f00.
00010000:00000010:1.1:1787760920.988472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479e600.
00000080:00000010:3.1:1787760921.027174:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca8e0.
00000080:00000010:3.1:1787760921.027192:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf740.
00010000:00000010:3.1:1787760921.745188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466d00.
00010000:00000010:3.1:1787760921.745202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479e800.
00000080:00000010:3.1:1787760921.791262:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811df00.
00000080:00000010:3.1:1787760921.791338:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf240.
00010000:00000010:2.1:1787760922.561540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518d800.
00010000:00000010:2.1:1787760922.561559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3d200.
00000080:00000010:3.1:1787760922.585671:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65140.
00000080:00000010:3.1:1787760922.585690:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d3e0.
00010000:00000010:3.1:1787760923.537391:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466900.
00010000:00000010:3.1:1787760923.537419:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8762e00.
00000080:00000010:3.1:1787760923.646356:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d3e0.
00000080:00000010:3.1:1787760923.646382:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca1c0.
00010000:00000010:3.1:1787760924.457769:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466100.
00010000:00000010:3.1:1787760924.457793:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479e800.
00000080:00000010:3.1:1787760924.548176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d3e0.
00000080:00000010:3.1:1787760924.548191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf020.
00010000:00000010:3.1:1787760924.980292:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466000.
00010000:00000010:3.1:1787760924.980307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479f400.
00000080:00000010:0.1:1787760925.033729:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d3e0.
00000080:00000010:0.1:1787760925.033748:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf6c0.
00010000:00000010:3.1:1787760925.469191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466a00.
00010000:00000010:3.1:1787760925.469206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479f800.
00000080:00000010:2.1:1787760925.567006:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811de00.
00000080:00000010:2.1:1787760925.567027:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf6c0.
00010000:00000010:1.1:1787760926.441191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786fa00.
00010000:00000010:1.1:1787760926.441205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980a00.
00000080:00000010:3.1:1787760926.459394:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf5a0.
00000080:00000010:3.1:1787760926.459408:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca760.
00010000:00000010:2.1:1787760927.462300:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786f600.
00010000:00000010:2.1:1787760927.462313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980a00.
00000080:00000010:3.1:1787760927.490728:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65e80.
00000080:00000010:3.1:1787760927.490747:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca8c0.
00010000:00000010:2.1:1787760928.435190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786f300.
00010000:00000010:2.1:1787760928.435205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479f800.
00000080:00000010:1.1:1787760928.520047:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65800.
00000080:00000010:1.1:1787760928.520061:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf480.
00010000:00000010:0.1:1787760929.472628:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873d000.
00010000:00000010:0.1:1787760929.472643:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3c000.
00000080:00000010:0.1:1787760929.558117:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf6a0.
00000080:00000010:0.1:1787760929.558151:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d060.
00010000:00000010:0.1:1787760930.257360:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466300.
00010000:00000010:0.1:1787760930.257375:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479e600.
00000080:00000010:1.1:1787760930.310238:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf800.
00000080:00000010:1.1:1787760930.310255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf6a0.
00010000:00000010:0.1:1787760930.891406:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786fb00.
00010000:00000010:0.1:1787760930.891422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980a00.
00000080:00000010:2.1:1787760930.978642:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf9c0.
00000080:00000010:2.1:1787760930.978661:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca960.
00010000:00000010:0.1:1787760931.243230:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786fc00.
00010000:00000010:0.1:1787760931.243244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3dc00.
00000080:00000010:0.1:1787760931.305277:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfc20.
00000080:00000010:0.1:1787760931.305289:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d6a0.
00010000:00000010:3.1:1787760931.658025:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786fe00.
00010000:00000010:3.1:1787760931.658041:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981200.
00000080:00000010:3.1:1787760931.662845:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811ddc0.
00000080:00000010:3.1:1787760931.662861:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfc20.
00010000:00000010:1.1:1787760932.125338:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786ff00.
00010000:00000010:1.1:1787760932.125349:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981000.
00000080:00000010:1.1:1787760932.125354:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca5e0.
00000080:00000010:1.1:1787760932.125362:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caec0.
00010000:00000010:2.1:1787760932.813072:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466a00.
00010000:00000010:2.1:1787760932.813086:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479f400.
00000080:00000010:0.1:1787760932.871176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65dc0.
00000080:00000010:0.1:1787760932.871191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf1e0.
00010000:00000010:0.1:1787760933.882201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786ff00.
00010000:00000010:0.1:1787760933.882215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981000.
00000080:00000010:0.1:1787760933.967014:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf3c0.
00000080:00000010:0.1:1787760933.967028:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caec0.
00010000:00000010:2.1:1787760934.398179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466100.
00010000:00000010:2.1:1787760934.398194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876a800.
00000080:00000010:0.1:1787760934.434184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65fc0.
00000080:00000010:0.1:1787760934.434198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65dc0.
00010000:00000010:0.1:1787760934.621174:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466d00.
00010000:00000010:0.1:1787760934.621185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831e00.
00000080:00000010:1.1:1787760934.656249:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfbe0.
00000080:00000010:1.1:1787760934.656267:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf180.
00010000:00000010:1.1:1787760934.813270:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786f900.
00010000:00000010:1.1:1787760934.813351:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980a00.
00000080:00000010:1.1:1787760934.838376:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cac00.
00000080:00000010:1.1:1787760934.838388:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cac60.
00010000:00000010:2.1:1787760935.082361:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786f400.
00010000:00000010:2.1:1787760935.082376:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d7400.
00000080:00000010:3.1:1787760935.148484:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65760.
00000080:00000010:3.1:1787760935.148504:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cac60.
00010000:00000010:3.1:1787760935.610482:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873da00.
00010000:00000010:3.1:1787760935.610493:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3de00.
00000080:00000010:3.1:1787760935.610498:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d6a0.
00000080:00000010:3.1:1787760935.610506:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d940.
00010000:00000010:1.1:1787760935.953671:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786ff00.
00010000:00000010:1.1:1787760935.953680:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d7a00.
00000080:00000010:1.1:1787760935.983254:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca360.
00000080:00000010:1.1:1787760935.983265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca9a0.
00010000:00000010:3.1:1787760936.196847:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646400.
00010000:00000010:3.1:1787760936.196863:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d7000.
00000080:00000010:3.1:1787760936.213174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d940.
00000080:00000010:3.1:1787760936.213188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca9a0.
00010000:00000010:3.1:1787760936.469208:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786fc00.
00010000:00000010:3.1:1787760936.469225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d6800.
00000080:00000010:3.1:1787760936.514194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d940.
00000080:00000010:3.1:1787760936.514212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca360.
00010000:00000010:3.1:1787760936.972425:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466f00.
00010000:00000010:3.1:1787760936.972467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7830c00.
00000080:00000010:3.1:1787760937.049182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d940.
00000080:00000010:3.1:1787760937.049200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf180.
00010000:00000010:0.1:1787760937.435198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873d600.
00010000:00000010:0.1:1787760937.435210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7830200.
00000080:00000010:0.1:1787760937.468186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfc20.
00000080:00000010:0.1:1787760937.468204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf1e0.
00010000:00000010:2.1:1787760938.038182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466100.
00010000:00000010:2.1:1787760938.038197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7830200.
00000080:00000010:0.1:1787760938.085131:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b659e0.
00000080:00000010:0.1:1787760938.085165:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf1e0.
00010000:00000010:1.1:1787760938.432439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466100.
00010000:00000010:1.1:1787760938.432454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831800.
00000080:00000010:1.1:1787760938.450738:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cabc0.
00000080:00000010:1.1:1787760938.450753:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf1e0.
00010000:00000010:2.1:1787760938.951462:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646900.
00010000:00000010:2.1:1787760938.951478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3dc00.
00000080:00000010:0.1:1787760939.002182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65d60.
00000080:00000010:0.1:1787760939.002198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d060.
00010000:00000010:2.1:1787760939.397408:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873da00.
00010000:00000010:2.1:1787760939.397423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876a600.
00000080:00000010:0.1:1787760939.418216:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65920.
00000080:00000010:0.1:1787760939.418231:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b657c0.
00010000:00000010:1.1:1787760939.904213:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646500.
00010000:00000010:1.1:1787760939.904230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3c000.
00000080:00000010:1.1:1787760939.927208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cabc0.
00000080:00000010:1.1:1787760939.927226:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811de00.
00010000:00000010:2.1:1787760940.423183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d900.
00010000:00000010:2.1:1787760940.423200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7830e00.
00000080:00000010:0.1:1787760940.481895:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b653a0.
00000080:00000010:0.1:1787760940.481917:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf8a0.
00010000:00000010:2.1:1787760940.859191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786fb00.
00010000:00000010:2.1:1787760940.859205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831a00.
00000080:00000010:0.1:1787760940.894748:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65880.
00000080:00000010:0.1:1787760940.894761:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf8a0.
00010000:00000010:1.1:1787760941.107493:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d200.
00010000:00000010:1.1:1787760941.107507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7830800.
00000080:00000010:1.1:1787760941.114412:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cabc0.
00000080:00000010:1.1:1787760941.114428:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf8a0.
00010000:00000010:1.1:1787760941.311177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d700.
00010000:00000010:1.1:1787760941.311192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831200.
00000080:00000010:1.1:1787760941.340183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca780.
00000080:00000010:1.1:1787760941.340198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf480.
00010000:00000010:0.1:1787760941.527201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786f600.
00010000:00000010:0.1:1787760941.527217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d6000.
00000080:00000010:0.1:1787760941.544197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf380.
00000080:00000010:0.1:1787760941.544212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca780.
00010000:00000010:0.1:1787760942.007685:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646e00.
00010000:00000010:0.1:1787760942.007702:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3d200.
00000080:00000010:0.1:1787760942.048171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf380.
00000080:00000010:0.1:1787760942.048185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d560.
00010000:00000010:0.1:1787760942.533371:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873d000.
00010000:00000010:0.1:1787760942.533386:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876bc00.
00000080:00000010:0.1:1787760942.544167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf380.
00000080:00000010:0.1:1787760942.544179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65720.
00010000:00000010:3.1:1787760942.953566:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7da00.
00010000:00000010:3.1:1787760942.953582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3ce00.
00000080:00000010:0.1:1787760942.972564:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d280.
00000080:00000010:0.1:1787760942.972584:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811de60.
00010000:00000010:2.1:1787760943.609412:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d600.
00010000:00000010:2.1:1787760943.609428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7830000.
00000080:00000010:2.1:1787760943.708186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65340.
00000080:00000010:2.1:1787760943.708204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf5a0.
00010000:00000010:1.1:1787760944.223637:0:20:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873df00.
00010000:00000010:1.1:1787760944.223652:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876a400.
00000080:00000010:1.1:1787760944.287576:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca060.
00000080:00000010:1.1:1787760944.287598:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65340.
00010000:00000010:1.1:1787760944.849179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873d300.
00010000:00000010:1.1:1787760944.849196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876a400.
00000080:00000010:1.1:1787760944.917836:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca060.
00000080:00000010:1.1:1787760944.917918:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65ba0.
00010000:00000010:2.1:1787760945.454379:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873d300.
00010000:00000010:2.1:1787760945.454398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876bc00.
00000080:00000010:0.1:1787760945.478176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65d80.
00000080:00000010:0.1:1787760945.478194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65a60.
00010000:00000010:3.1:1787760946.023739:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873df00.
00010000:00000010:3.1:1787760946.023755:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876b000.
00000080:00000010:0.1:1787760946.093127:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811dde0.
00000080:00000010:0.1:1787760946.093159:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65bc0.
00010000:00000010:0.1:1787760946.582712:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d300.
00010000:00000010:0.1:1787760946.582726:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3d400.
00000080:00000010:0.1:1787760946.615173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf020.
00000080:00000010:0.1:1787760946.615184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d6e0.
00010000:00000010:0.1:1787760946.993231:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646800.
00010000:00000010:0.1:1787760946.993247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3c200.
00000080:00000010:0.1:1787760947.007184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf020.
00000080:00000010:0.1:1787760947.007197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d9c0.
00010000:00000010:1.1:1787760947.207175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d300.
00010000:00000010:1.1:1787760947.207191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876b000.
00000080:00000010:1.1:1787760947.235019:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca060.
00000080:00000010:1.1:1787760947.235029:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65480.
00010000:00000010:1.1:1787760947.495175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d800.
00010000:00000010:1.1:1787760947.495191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d6c00.
00000080:00000010:1.1:1787760947.520177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cada0.
00000080:00000010:1.1:1787760947.520192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca060.
00010000:00000010:1.1:1787760947.752978:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7dc00.
00010000:00000010:1.1:1787760947.752992:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831600.
00000080:00000010:1.1:1787760947.791557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca060.
00000080:00000010:1.1:1787760947.791570:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf020.
00010000:00000010:3.1:1787760948.123253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7de00.
00010000:00000010:3.1:1787760948.123269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831200.
00000080:00000010:0.1:1787760948.178788:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d220.
00000080:00000010:0.1:1787760948.178808:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf240.
00010000:00000010:2.1:1787760948.474895:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d500.
00010000:00000010:2.1:1787760948.474913:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831a00.
00000080:00000010:0.1:1787760948.541741:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b659e0.
00000080:00000010:0.1:1787760948.541755:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf240.
00010000:00000010:3.1:1787760948.907365:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d500.
00010000:00000010:3.1:1787760948.907380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876ba00.
00000080:00000010:3.1:1787760948.919509:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d220.
00000080:00000010:3.1:1787760948.919529:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b653e0.
00010000:00000010:0.1:1787760949.107631:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7dc00.
00010000:00000010:0.1:1787760949.107639:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831800.
00000080:00000010:0.1:1787760949.127900:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfa40.
00000080:00000010:0.1:1787760949.127916:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfa20.
00010000:00000010:2.1:1787760949.640447:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d800.
00010000:00000010:2.1:1787760949.640455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb43a00.
00000080:00000010:3.1:1787760949.699903:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65780.
00000080:00000010:3.1:1787760949.699917:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfa20.
00010000:00000010:2.1:1787760950.466612:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d300.
00010000:00000010:2.1:1787760950.466628:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42800.
00000080:00000010:3.1:1787760950.467404:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65420.
00000080:00000010:3.1:1787760950.467417:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65ae0.
00010000:00000010:2.1:1787760951.136463:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d600.
00010000:00000010:2.1:1787760951.136477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42800.
00000080:00000010:0.1:1787760951.201927:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65760.
00000080:00000010:0.1:1787760951.201956:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65500.
00010000:00000010:3.1:1787760951.935555:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d700.
00010000:00000010:3.1:1787760951.935571:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831800.
00000080:00000010:2.1:1787760951.995826:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d9c0.
00000080:00000010:2.1:1787760951.995852:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfa40.
00010000:00000010:1.1:1787760952.436580:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4800.
00010000:00000010:1.1:1787760952.436589:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d7800.
00000080:00000010:1.1:1787760952.461844:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caa20.
00000080:00000010:1.1:1787760952.461862:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caf00.
00010000:00000010:1.1:1787760953.036665:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d900.
00010000:00000010:1.1:1787760953.036681:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d6400.
00000080:00000010:1.1:1787760953.100437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caa20.
00000080:00000010:1.1:1787760953.100457:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caf00.
00010000:00000010:3.1:1787760953.688108:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d700.
00010000:00000010:3.1:1787760953.688124:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831800.
00000080:00000010:3.1:1787760953.715729:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d6e0.
00000080:00000010:3.1:1787760953.715748:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf1c0.
00010000:00000010:3.1:1787760954.309180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646a00.
00010000:00000010:3.1:1787760954.309195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3d200.
00000080:00000010:3.1:1787760954.375430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811de60.
00000080:00000010:3.1:1787760954.375454:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811dde0.
00010000:00000010:0.1:1787760954.587178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646700.
00010000:00000010:0.1:1787760954.587196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3c000.
00000080:00000010:0.1:1787760954.624163:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf7a0.
00000080:00000010:0.1:1787760954.624177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811dde0.
00010000:00000010:2.1:1787760955.018178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646700.
00010000:00000010:2.1:1787760955.018192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3c000.
00000080:00000010:0.1:1787760955.066182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65d60.
00000080:00000010:0.1:1787760955.066195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d560.
00010000:00000010:2.1:1787760955.449185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d300.
00010000:00000010:2.1:1787760955.449201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42000.
00000080:00000010:0.1:1787760955.611791:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65920.
00000080:00000010:0.1:1787760955.611811:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65480.
00010000:00000010:3.1:1787760956.704715:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d500.
00010000:00000010:3.1:1787760956.704738:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7830e00.
00000080:00000010:3.1:1787760956.792322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d060.
00000080:00000010:3.1:1787760956.792350:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf6e0.
00010000:00000010:0.1:1787760957.683956:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d300.
00010000:00000010:0.1:1787760957.683974:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42600.
00000080:00000010:2.1:1787760957.982294:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfd20.
00000080:00000010:2.1:1787760957.982304:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65a20.
00010000:00000010:2.1:1787760959.424241:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d700.
00010000:00000010:2.1:1787760959.424251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831200.
00000080:00000010:0.1:1787760959.439835:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65b80.
00000080:00000010:0.1:1787760959.444211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf600.
00010000:00000010:2.1:1787760959.712303:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1be00.
00010000:00000010:2.1:1787760959.712474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831600.
00000080:00000010:2.1:1787760959.724525:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65b40.
00000080:00000010:2.1:1787760959.724537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf600.
00010000:00000010:2.1:1787760959.932174:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1bf00.
00010000:00000010:2.1:1787760959.932199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42c00.
00000080:00000010:0.1:1787760959.972486:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65200.
00000080:00000010:0.1:1787760959.972503:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65a60.
00010000:00000010:3.1:1787760960.264335:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4c00.
00010000:00000010:3.1:1787760960.264345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d6400.
00000080:00000010:3.1:1787760960.276928:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d560.
00000080:00000010:3.1:1787760960.284160:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caa20.
00010000:00000010:2.1:1787760960.863670:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4f00.
00010000:00000010:2.1:1787760960.863686:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d6000.
00000080:00000010:0.1:1787760960.900370:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65ea0.
00000080:00000010:0.1:1787760960.900385:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca9e0.
00010000:00000010:2.1:1787760961.496988:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4d00.
00010000:00000010:2.1:1787760961.497009:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d7200.
00000080:00000010:0.1:1787760961.516173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65200.
00000080:00000010:0.1:1787760961.516189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca120.
00010000:00000010:1.1:1787760962.026223:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646e00.
00010000:00000010:1.1:1787760962.026235:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d7000.
00000080:00000010:1.1:1787760962.077322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca500.
00000080:00000010:1.1:1787760962.077341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca440.
00010000:00000010:2.1:1787760962.356180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1bd00.
00010000:00000010:2.1:1787760962.356194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42600.
00000080:00000010:0.1:1787760962.410472:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b653a0.
00000080:00000010:0.1:1787760962.410491:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65880.
00010000:00000010:1.1:1787760962.664489:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4200.
00010000:00000010:1.1:1787760962.664508:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d7200.
00000080:00000010:1.1:1787760962.708882:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caea0.
00000080:00000010:1.1:1787760962.708897:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca500.
00010000:00000010:3.1:1787760963.044063:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4000.
00010000:00000010:3.1:1787760963.044074:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d6400.
00000080:00000010:0.1:1787760963.058170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e811d560.
00000080:00000010:0.1:1787760963.058184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca500.
00010000:00000010:0.1:1787760963.464253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1b400.
00010000:00000010:0.1:1787760963.464267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7830e00.
00000080:00000010:0.1:1787760963.514190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acff80.
00000080:00000010:0.1:1787760963.514215:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf760.
00010000:00000010:0.1:1787760963.766341:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4a00.
00010000:00000010:0.1:1787760963.766356:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d6000.
00000080:00000010:0.1:1787760963.812324:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf760.
00000080:00000010:0.1:1787760963.812334:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caba0.
00010000:00000010:0.1:1787760964.092616:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1b100.
00010000:00000010:0.1:1787760964.092630:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb43800.
00000080:00000010:0.1:1787760964.105185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf760.
00000080:00000010:0.1:1787760964.105206:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65860.
00010000:00000010:0.1:1787760964.530187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4300.
00010000:00000010:0.1:1787760964.530211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fd200.
00000080:00000010:1.1:1787760964.591182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf760.
00000080:00000010:1.1:1787760964.591210:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caa60.
00010000:00000010:1.1:1787760965.212186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4a00.
00010000:00000010:1.1:1787760965.212214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fc000.
00000080:00000010:1.1:1787760965.312839:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca280.
00000080:00000010:1.1:1787760965.312852:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caa60.
00010000:00000010:1.1:1787760965.684178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646900.
00010000:00000010:1.1:1787760965.684210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42600.
00000080:00000010:1.1:1787760965.716085:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caa60.
00000080:00000010:1.1:1787760965.716100:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65f60.
00010000:00000010:2.1:1787760966.368464:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1b200.
00010000:00000010:2.1:1787760966.368479:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831800.
00000080:00000010:0.1:1787760966.388408:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65ac0.
00000080:00000010:0.1:1787760966.388421:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf440.
00010000:00000010:0.1:1787760966.842200:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1b700.
00010000:00000010:0.1:1787760966.842215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831800.
00000080:00000010:0.1:1787760966.883161:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf520.
00000080:00000010:0.1:1787760966.883173:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf440.
00010000:00000010:1.1:1787760967.098492:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1ba00.
00010000:00000010:1.1:1787760967.098500:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0c00.
00000080:00000010:1.1:1787760967.102447:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caa60.
00000080:00000010:1.1:1787760967.102462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf520.
00010000:00000010:0.1:1787760967.380256:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1b300.
00010000:00000010:0.1:1787760967.380271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0200.
00000080:00000010:3.1:1787760967.422192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf7c0.
00000080:00000010:3.1:1787760967.426606:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf9e0.
00010000:00000010:0.1:1787760967.711762:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646200.
00010000:00000010:0.1:1787760967.711779:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8777c00.
00000080:00000010:1.1:1787760967.798855:0:1365:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf620.
00000080:00000010:1.1:1787760967.798871:0:1365:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751cfd29bc0.
00010000:00000010:1.1:1787760968.406180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1b800.
00010000:00000010:1.1:1787760968.406192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0200.
00000080:00000010:1.1:1787760968.465291:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caa60.
00000080:00000010:1.1:1787760968.465311:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf140.
00010000:00000010:2.1:1787760969.548561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646900.
00010000:00000010:2.1:1787760969.548577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8777e00.
00000080:00000010:1.1:1787760969.634388:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65ac0.
00000080:00000010:1.1:1787760969.752574:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751cfd29bc0.
00010000:00000010:2.1:1787760970.756508:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646700.
00010000:00000010:2.1:1787760970.756542:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8776a00.
00000080:00000010:0.1:1787760970.810186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65860.
00000080:00000010:0.1:1787760970.810200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c302b5c0.
00010000:00000010:2.1:1787760971.351178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1bc00.
00010000:00000010:2.1:1787760971.351193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0400.
00000080:00000010:0.1:1787760971.438007:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b659e0.
00000080:00000010:0.1:1787760971.438027:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfe80.
00010000:00000010:2.1:1787760972.207339:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646700.
00010000:00000010:2.1:1787760972.207355:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8777800.
00000080:00000010:2.1:1787760972.273320:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b657c0.
00000080:00000010:2.1:1787760972.273334:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28800.
00010000:00000010:1.1:1787760972.837192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1b800.
00010000:00000010:1.1:1787760972.837206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c1400.
00000080:00000010:1.1:1787760972.892304:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca1e0.
00000080:00000010:1.1:1787760972.892320:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfe80.
00010000:00000010:1.1:1787760973.438441:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1ba00.
00010000:00000010:1.1:1787760973.438449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42200.
00000080:00000010:3.1:1787760973.458379:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca1e0.
00000080:00000010:3.1:1787760973.458395:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65b80.
00010000:00000010:0.1:1787760974.164180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4d00.
00010000:00000010:0.1:1787760974.164194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fca00.
00000080:00000010:0.1:1787760974.214270:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf2c0.
00000080:00000010:0.1:1787760974.214281:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cae20.
00010000:00000010:0.1:1787760974.431766:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1bd00.
00010000:00000010:0.1:1787760974.431778:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c1000.
00000080:00000010:0.1:1787760974.466281:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf0e0.
00000080:00000010:0.1:1787760974.466295:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf2c0.
00010000:00000010:2.1:1787760974.694999:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4f00.
00010000:00000010:2.1:1787760974.695015:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0800.
00000080:00000010:1.1:1787760974.718929:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65740.
00000080:00000010:1.1:1787760974.718946:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf2c0.
00010000:00000010:0.1:1787760975.016456:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1be00.
00010000:00000010:0.1:1787760975.016478:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0600.
00000080:00000010:0.1:1787760975.032621:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf2c0.
00000080:00000010:0.1:1787760975.032632:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf0e0.
00010000:00000010:2.1:1787760975.574443:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4c00.
00010000:00000010:2.1:1787760975.574459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fc200.
00000080:00000010:2.1:1787760975.596300:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65d80.
00000080:00000010:2.1:1787760975.596315:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca680.
00010000:00000010:3.1:1787760976.406391:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2e00.
00010000:00000010:3.1:1787760976.406408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42600.
00000080:00000010:0.1:1787760976.493727:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28220.
00000080:00000010:0.1:1787760976.493745:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65b40.
00010000:00000010:0.1:1787760977.202213:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646700.
00010000:00000010:0.1:1787760977.202232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8776e00.
00000080:00000010:1.1:1787760977.236518:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfe80.
00000080:00000010:1.1:1787760977.236534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28e60.
00010000:00000010:3.1:1787760977.782293:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518df00.
00010000:00000010:3.1:1787760977.782311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8777000.
00000080:00000010:3.1:1787760977.822848:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28e80.
00000080:00000010:3.1:1787760977.822863:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28f20.
00010000:00000010:2.1:1787760978.182421:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4b00.
00010000:00000010:2.1:1787760978.182435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c1600.
00000080:00000010:0.1:1787760978.275784:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65f60.
00000080:00000010:0.1:1787760978.275806:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf860.
00010000:00000010:2.1:1787760978.735183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646700.
00010000:00000010:2.1:1787760978.735199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42400.
00000080:00000010:0.1:1787760978.772171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65ac0.
00000080:00000010:0.1:1787760978.772192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf860.
00010000:00000010:2.1:1787760979.244490:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646700.
00010000:00000010:2.1:1787760979.244507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42200.
00000080:00000010:2.1:1787760979.355641:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65ec0.
00000080:00000010:2.1:1787760979.355673:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65f80.
00010000:00000010:0.1:1787760979.583178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2f00.
00010000:00000010:0.1:1787760979.583191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8776800.
00000080:00000010:0.1:1787760979.651039:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf860.
00000080:00000010:0.1:1787760979.651052:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28e80.
00010000:00000010:0.1:1787760979.918862:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2d00.
00010000:00000010:0.1:1787760979.918877:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42c00.
00000080:00000010:0.1:1787760979.950180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf860.
00000080:00000010:0.1:1787760979.950194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65ae0.
00010000:00000010:3.1:1787760980.292451:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1a900.
00010000:00000010:3.1:1787760980.292466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c1e00.
00000080:00000010:0.1:1787760980.344359:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28c20.
00000080:00000010:0.1:1787760980.344373:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf860.
00010000:00000010:0.1:1787760980.750213:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4c00.
00010000:00000010:0.1:1787760980.750227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c1600.
00000080:00000010:0.1:1787760980.794203:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf9e0.
00000080:00000010:0.1:1787760980.794222:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf620.
00010000:00000010:3.1:1787760981.201240:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2400.
00010000:00000010:3.1:1787760981.201255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42400.
00000080:00000010:0.1:1787760981.229255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28e00.
00000080:00000010:0.1:1787760981.229269:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65da0.
00010000:00000010:1.1:1787760981.816263:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2200.
00010000:00000010:1.1:1787760981.816277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42c00.
00000080:00000010:1.1:1787760981.883179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca020.
00000080:00000010:1.1:1787760981.883192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65fc0.
00010000:00000010:0.1:1787760982.358202:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2700.
00010000:00000010:0.1:1787760982.358219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3c000.
00000080:00000010:2.1:1787760982.429170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf9e0.
00000080:00000010:2.1:1787760982.429187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65760.
00010000:00000010:2.1:1787760982.990182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4f00.
00010000:00000010:2.1:1787760982.990195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0600.
00000080:00000010:0.1:1787760983.029208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65760.
00000080:00000010:0.1:1787760983.029220:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf7c0.
00010000:00000010:1.1:1787760983.237716:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1a000.
00010000:00000010:1.1:1787760983.237730:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c1400.
00000080:00000010:1.1:1787760983.265800:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca020.
00000080:00000010:1.1:1787760983.265819:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf7c0.
00010000:00000010:2.1:1787760983.453270:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1aa00.
00010000:00000010:2.1:1787760983.453286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0400.
00000080:00000010:0.1:1787760983.483537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65700.
00000080:00000010:0.1:1787760983.483557:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf600.
00010000:00000010:1.1:1787760984.113367:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4d00.
00010000:00000010:1.1:1787760984.113382:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fd800.
00000080:00000010:1.1:1787760984.158176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca000.
00000080:00000010:1.1:1787760984.158195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca020.
00010000:00000010:3.1:1787760984.765306:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1a300.
00010000:00000010:3.1:1787760984.765319:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fc400.
00000080:00000010:3.1:1787760984.810482:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28400.
00000080:00000010:3.1:1787760984.810497:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca020.
00010000:00000010:3.1:1787760985.459520:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1ab00.
00010000:00000010:3.1:1787760985.459535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0200.
00000080:00000010:3.1:1787760985.517221:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28400.
00000080:00000010:3.1:1787760985.517247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf600.
00010000:00000010:2.1:1787760986.031183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1ae00.
00010000:00000010:2.1:1787760986.031199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0c00.
00000080:00000010:0.1:1787760986.047218:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65d80.
00000080:00000010:0.1:1787760986.047306:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf6e0.
00010000:00000010:1.1:1787760986.502291:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1a500.
00010000:00000010:1.1:1787760986.502305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0200.
00000080:00000010:1.1:1787760986.535176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca000.
00000080:00000010:1.1:1787760986.535188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf1c0.
00010000:00000010:3.1:1787760986.931387:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2600.
00010000:00000010:3.1:1787760986.931403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a9000.
00000080:00000010:2.1:1787760986.947992:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28400.
00000080:00000010:2.1:1787760986.948020:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65720.
00010000:00000010:2.1:1787760987.521217:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4f00.
00010000:00000010:2.1:1787760987.521235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fd400.
00000080:00000010:2.1:1787760987.609400:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65d80.
00000080:00000010:2.1:1787760987.609421:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65340.
00010000:00000010:2.1:1787760988.214298:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4c00.
00010000:00000010:2.1:1787760988.214313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c1400.
00000080:00000010:2.1:1787760988.264430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65d80.
00000080:00000010:2.1:1787760988.264444:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfa40.
00010000:00000010:2.1:1787760988.684289:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2700.
00010000:00000010:2.1:1787760988.684298:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a8200.
00000080:00000010:2.1:1787760988.701439:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65c20.
00000080:00000010:2.1:1787760988.701452:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65d80.
00010000:00000010:2.1:1787760989.157176:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2200.
00010000:00000010:2.1:1787760989.157190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a9800.
00000080:00000010:0.1:1787760989.173287:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65880.
00000080:00000010:0.1:1787760989.173305:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65c20.
00010000:00000010:3.1:1787760989.823190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4b00.
00010000:00000010:3.1:1787760989.823204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0200.
00000080:00000010:3.1:1787760989.846211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac284e0.
00000080:00000010:3.1:1787760989.846225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf240.
00010000:00000010:2.1:1787760990.360263:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4c00.
00010000:00000010:2.1:1787760990.360273:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fda00.
00000080:00000010:0.1:1787760990.386211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65d60.
00000080:00000010:0.1:1787760990.386229:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca940.
00010000:00000010:3.1:1787760990.845340:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5ec00.
00010000:00000010:3.1:1787760990.845370:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8777200.
00000080:00000010:3.1:1787760990.873165:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28400.
00000080:00000010:3.1:1787760990.873186:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28d60.
00010000:00000010:1.1:1787760991.181242:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4c00.
00010000:00000010:1.1:1787760991.181257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fd000.
00000080:00000010:1.1:1787760991.224365:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca020.
00000080:00000010:1.1:1787760991.224380:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca940.
00010000:00000010:2.1:1787760991.597172:0:257570:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1ae00.
00010000:00000010:2.1:1787760991.597180:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a8600.
00000080:00000010:0.1:1787760991.662181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65ae0.
00000080:00000010:0.1:1787760991.662194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65ac0.
00010000:00000010:1.1:1787760992.126606:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5ef00.
00010000:00000010:1.1:1787760992.126621:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8776e00.
00000080:00000010:2.1:1787760992.186340:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca940.
00000080:00000010:2.1:1787760992.186358:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28c20.
00010000:00000010:0.1:1787760992.841585:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1a300.
00010000:00000010:0.1:1787760992.841601:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0200.
00000080:00000010:0.1:1787760992.852652:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf380.
00000080:00000010:0.1:1787760992.852671:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf5a0.
00010000:00000010:2.1:1787760993.293317:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5ed00.
00010000:00000010:2.1:1787760993.293331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8776a00.
00000080:00000010:0.1:1787760993.344329:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65700.
00000080:00000010:0.1:1787760993.354233:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28e60.
00010000:00000010:3.1:1787760993.621306:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870ce00.
00010000:00000010:3.1:1787760993.621317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3c000.
00000080:00000010:2.1:1787760993.644840:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28880.
00000080:00000010:2.1:1787760993.644860:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf380.
00010000:00000010:3.1:1787760993.906447:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2400.
00010000:00000010:3.1:1787760993.906472:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a8000.
00000080:00000010:3.1:1787760993.920848:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28220.
00000080:00000010:3.1:1787760993.920868:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b659e0.
00010000:00000010:1.1:1787760994.388704:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1a000.
00010000:00000010:1.1:1787760994.388724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3c000.
00000080:00000010:1.1:1787760994.430621:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca020.
00000080:00000010:1.1:1787760994.430645:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf8a0.
00010000:00000010:0.1:1787760994.711175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1a000.
00010000:00000010:0.1:1787760994.711189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1b400.
00000080:00000010:0.1:1787760994.739182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf180.
00000080:00000010:0.1:1787760994.739195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf1e0.
00010000:00000010:2.1:1787760994.948969:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870cf00.
00010000:00000010:2.1:1787760994.949053:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fc800.
00000080:00000010:0.1:1787760994.983178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65940.
00000080:00000010:0.1:1787760994.983188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf1e0.
00010000:00000010:2.1:1787760995.280375:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98d00.
00010000:00000010:2.1:1787760995.280390:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1ba00.
00000080:00000010:1.1:1787760995.322730:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65080.
00000080:00000010:1.1:1787760995.322743:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf1e0.
00010000:00000010:3.1:1787760995.999535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98400.
00010000:00000010:3.1:1787760995.999552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1b000.
00000080:00000010:3.1:1787760996.025216:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28220.
00000080:00000010:3.1:1787760996.025231:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf1e0.
00010000:00000010:2.1:1787760996.506203:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c500.
00010000:00000010:2.1:1787760996.506219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1a800.
00000080:00000010:1.1:1787760996.544267:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65920.
00000080:00000010:1.1:1787760996.544282:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf8a0.
00010000:00000010:1.1:1787760997.197194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98000.
00010000:00000010:1.1:1787760997.197211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1b200.
00000080:00000010:1.1:1787760997.212298:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca020.
00000080:00000010:1.1:1787760997.212311:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf380.
00010000:00000010:2.1:1787760997.542595:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98a00.
00010000:00000010:2.1:1787760997.542606:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1bc00.
00000080:00000010:2.1:1787760997.547708:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65480.
00000080:00000010:2.1:1787760997.547722:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf240.
00010000:00000010:3.1:1787760998.008829:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c900.
00010000:00000010:3.1:1787760998.008842:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fc800.
00000080:00000010:3.1:1787760998.025174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28800.
00000080:00000010:3.1:1787760998.059692:0:257571:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cae20.
00010000:00000010:3.1:1787760998.364832:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98600.
00010000:00000010:3.1:1787760998.364849:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fd400.
00000080:00000010:1.1:1787760998.409418:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28800.
00000080:00000010:1.1:1787760998.409436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caa60.
00010000:00000010:2.1:1787760998.855177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98300.
00010000:00000010:2.1:1787760998.855190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1be00.
00000080:00000010:2.1:1787760998.877181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b653a0.
00000080:00000010:2.1:1787760998.877195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfa40.
00010000:00000010:0.1:1787760999.263754:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c200.
00010000:00000010:0.1:1787760999.263769:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fc200.
00000080:00000010:0.1:1787760999.302812:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf7c0.
00000080:00000010:0.1:1787760999.343318:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf600.
00010000:00000010:3.1:1787760999.755787:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5e900.
00010000:00000010:3.1:1787760999.755801:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a9000.
00000080:00000010:0.1:1787760999.837211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28e60.
00000080:00000010:0.1:1787760999.837226:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65200.
00010000:00000010:2.1:1787761000.296475:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c700.
00010000:00000010:2.1:1787761000.296489:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fdc00.
00000080:00000010:3.1:1787761000.318636:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b659e0.
00000080:00000010:3.1:1787761000.318654:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca500.
00010000:00000010:2.1:1787761000.757187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98c00.
00010000:00000010:2.1:1787761000.757202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1a200.
00000080:00000010:0.1:1787761000.776171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65760.
00000080:00000010:0.1:1787761000.776187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf7c0.
00010000:00000010:3.1:1787761001.168244:0:13:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98e00.
00010000:00000010:3.1:1787761001.168260:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8776e00.
00000080:00000010:3.1:1787761001.187304:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28800.
00000080:00000010:3.1:1787761001.187316:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750fac28e60.
00010000:00000010:2.1:1787761001.516216:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98f00.
00010000:00000010:2.1:1787761001.516232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a8600.
00000080:00000010:3.1:1787761001.544177:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65da0.
00000080:00000010:3.1:1787761001.544191:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b657e0.
00010000:00000010:0.1:1787761001.811186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98f00.
00010000:00000010:0.1:1787761001.811202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1a400.
00000080:00000010:0.1:1787761001.833177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfe80.
00000080:00000010:0.1:1787761001.833192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf9e0.
00010000:00000010:0.1:1787761002.306505:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98f00.
00010000:00000010:0.1:1787761002.306520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8776e00.
00000080:00000010:0.1:1787761002.321649:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf9e0.
00000080:00000010:0.1:1787761002.321666:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65940.
00010000:00000010:2.1:1787761002.721199:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5e200.
00010000:00000010:2.1:1787761002.721215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a8600.
00000080:00000010:0.1:1787761002.784473:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65b40.
00000080:00000010:0.1:1787761002.784490:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65bc0.
00010000:00000010:0.1:1787761003.235180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5e700.
00010000:00000010:0.1:1787761003.235196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877fe00.
00000080:00000010:0.1:1787761003.246549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfe80.
00000080:00000010:0.1:1787761003.246566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345160.
00010000:00000010:2.1:1787761003.700879:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5e600.
00010000:00000010:2.1:1787761003.700892:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877ea00.
00000080:00000010:0.1:1787761003.761190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65f80.
00000080:00000010:0.1:1787761003.761209:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345940.
00010000:00000010:2.1:1787761004.241526:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870ca00.
00010000:00000010:2.1:1787761004.241543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a9c00.
00000080:00000010:0.1:1787761004.292189:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65da0.
00000080:00000010:0.1:1787761004.292202:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65ae0.
00010000:00000010:2.1:1787761004.676807:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98e00.
00010000:00000010:2.1:1787761004.676818:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1aa00.
00000080:00000010:0.1:1787761004.722163:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65fc0.
00000080:00000010:0.1:1787761004.722178:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf9e0.
00010000:00000010:2.1:1787761005.124189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98300.
00010000:00000010:2.1:1787761005.124195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1be00.
00000080:00000010:3.1:1787761005.142535:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65da0.
00000080:00000010:3.1:1787761005.142550:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf9e0.
00010000:00000010:2.1:1787761005.613975:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5e600.
00010000:00000010:2.1:1787761005.613992:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877e400.
00000080:00000010:3.1:1787761005.713225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65760.
00000080:00000010:3.1:1787761005.713240:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345b40.
00010000:00000010:2.1:1787761006.210399:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98000.
00010000:00000010:2.1:1787761006.210417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1bc00.
00000080:00000010:0.1:1787761006.234296:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65080.
00000080:00000010:0.1:1787761006.234310:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfa40.
00010000:00000010:1.1:1787761006.742208:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c600.
00010000:00000010:1.1:1787761006.742222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877ee00.
00000080:00000010:1.1:1787761006.772256:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca120.
00000080:00000010:1.1:1787761006.772276:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345920.
00010000:00000010:2.1:1787761007.390215:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c300.
00010000:00000010:2.1:1787761007.390227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a8600.
00000080:00000010:3.1:1787761007.427844:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65580.
00000080:00000010:3.1:1787761007.427878:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65480.
00010000:00000010:2.1:1787761007.905398:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c800.
00010000:00000010:2.1:1787761007.905425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fdc00.
00000080:00000010:0.1:1787761007.969909:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65bc0.
00000080:00000010:0.1:1787761007.969940:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca9e0.
00010000:00000010:0.1:1787761008.230043:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98000.
00010000:00000010:0.1:1787761008.230058:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1b600.
00000080:00000010:0.1:1787761008.246506:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf240.
00000080:00000010:0.1:1787761008.246521:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf1c0.
00010000:00000010:0.1:1787761008.620812:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98000.
00010000:00000010:0.1:1787761008.620821:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b8800.
00000080:00000010:0.1:1787761008.651179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf1c0.
00000080:00000010:0.1:1787761008.651199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65880.
00010000:00000010:1.1:1787761009.102195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98600.
00010000:00000010:1.1:1787761009.102210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1a800.
00000080:00000010:1.1:1787761009.170388:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caa20.
00000080:00000010:1.1:1787761009.170404:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf1c0.
00010000:00000010:3.1:1787761009.733298:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98e00.
00010000:00000010:3.1:1787761009.733315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1ba00.
00000080:00000010:3.1:1787761009.750177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63458e0.
00000080:00000010:3.1:1787761009.750198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfaa0.
00010000:00000010:3.1:1787761010.281034:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870cb00.
00010000:00000010:3.1:1787761010.281048:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fc200.
00000080:00000010:2.1:1787761010.305175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63458e0.
00000080:00000010:2.1:1787761010.305189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caa20.
00010000:00000010:0.1:1787761010.743410:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870cc00.
00010000:00000010:0.1:1787761010.743424:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1ba00.
00000080:00000010:0.1:1787761010.772212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf1e0.
00000080:00000010:0.1:1787761010.772231:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf8a0.
00010000:00000010:2.1:1787761011.372178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870cc00.
00010000:00000010:2.1:1787761011.372193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1ba00.
00000080:00000010:1.1:1787761011.414444:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65ae0.
00000080:00000010:1.1:1787761011.414472:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf8a0.
00010000:00000010:2.1:1787761011.926124:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646c00.
00010000:00000010:2.1:1787761011.926156:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b9400.
00000080:00000010:0.1:1787761011.980477:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65580.
00000080:00000010:0.1:1787761011.980498:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65780.
00010000:00000010:1.1:1787761012.370805:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310700.
00010000:00000010:1.1:1787761012.370823:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b9400.
00000080:00000010:1.1:1787761012.425184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caf00.
00000080:00000010:1.1:1787761012.425207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65480.
00010000:00000010:2.1:1787761012.848276:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646f00.
00010000:00000010:2.1:1787761012.848289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1b000.
00000080:00000010:2.1:1787761012.865163:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65920.
00000080:00000010:2.1:1787761012.865182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf1e0.
00010000:00000010:0.1:1787761013.312533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c800.
00010000:00000010:0.1:1787761013.312550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fc800.
00000080:00000010:0.1:1787761013.330069:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf8a0.
00000080:00000010:0.1:1787761013.330089:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca780.
00010000:00000010:3.1:1787761013.811380:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5ef00.
00010000:00000010:3.1:1787761013.811393:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877e800.
00000080:00000010:3.1:1787761013.860180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345480.
00000080:00000010:3.1:1787761013.860216:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345ce0.
00010000:00000010:3.1:1787761014.289464:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646500.
00010000:00000010:3.1:1787761014.289481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b8200.
00000080:00000010:3.1:1787761014.305321:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345ce0.
00000080:00000010:3.1:1787761014.305334:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65700.
00010000:00000010:0.1:1787761014.678761:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c300.
00010000:00000010:0.1:1787761014.678769:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fc800.
00000080:00000010:1.1:1787761014.715484:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfaa0.
00000080:00000010:1.1:1787761014.715494:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cabc0.
00010000:00000010:2.1:1787761015.098191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c700.
00010000:00000010:2.1:1787761015.098206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fdc00.
00000080:00000010:1.1:1787761015.129074:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65780.
00000080:00000010:1.1:1787761015.129084:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cabc0.
00010000:00000010:0.1:1787761015.508362:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c900.
00010000:00000010:0.1:1787761015.508379:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fc800.
00000080:00000010:0.1:1787761015.536185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf380.
00000080:00000010:0.1:1787761015.536200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cabc0.
00010000:00000010:2.1:1787761015.841741:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646400.
00010000:00000010:2.1:1787761015.841756:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1be00.
00000080:00000010:0.1:1787761015.864179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b653a0.
00000080:00000010:0.1:1787761015.864197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf380.
00010000:00000010:2.1:1787761016.169967:0:1365:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646200.
00010000:00000010:2.1:1787761016.169977:0:1365:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1be00.
00000080:00000010:0.1:1787761016.179104:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b653a0.
00000080:00000010:0.1:1787761016.179114:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf380.
00010000:00000010:3.1:1787761016.459181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870cf00.
00010000:00000010:3.1:1787761016.459198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87ce600.
00000080:00000010:2.1:1787761016.488119:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345ce0.
00000080:00000010:2.1:1787761016.488153:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca5e0.
00010000:00000010:2.1:1787761016.975186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870cf00.
00010000:00000010:2.1:1787761016.975202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cf200.
00000080:00000010:2.1:1787761016.995256:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65760.
00000080:00000010:2.1:1787761016.995267:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caec0.
00010000:00000010:1.1:1787761017.274183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646700.
00010000:00000010:1.1:1787761017.274197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1ac00.
00000080:00000010:1.1:1787761017.291175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca960.
00000080:00000010:1.1:1787761017.291190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf380.
00010000:00000010:2.1:1787761017.707191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310800.
00010000:00000010:2.1:1787761017.707205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b8800.
00000080:00000010:2.1:1787761017.731188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65ae0.
00000080:00000010:2.1:1787761017.731218:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65f80.
00010000:00000010:0.1:1787761018.186182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5ec00.
00010000:00000010:0.1:1787761018.186192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877e000.
00000080:00000010:0.1:1787761018.238199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf9e0.
00000080:00000010:0.1:1787761018.238222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63450e0.
00010000:00000010:3.1:1787761018.601299:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5e600.
00010000:00000010:3.1:1787761018.601308:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877e000.
00000080:00000010:3.1:1787761018.640227:0:257570:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345dc0.
00000080:00000010:3.1:1787761018.656359:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345a20.
00010000:00000010:0.1:1787761018.987194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646600.
00010000:00000010:0.1:1787761018.987210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1b000.
00000080:00000010:0.1:1787761019.010178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf380.
00000080:00000010:0.1:1787761019.010195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf9e0.
00010000:00000010:0.1:1787761019.589190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310b00.
00010000:00000010:0.1:1787761019.589205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b9400.
00000080:00000010:2.1:1787761019.614164:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf9e0.
00000080:00000010:2.1:1787761019.614180:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65ae0.
00010000:00000010:0.1:1787761020.096725:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c900.
00010000:00000010:0.1:1787761020.096740:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b9800.
00000080:00000010:0.1:1787761020.098172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf380.
00000080:00000010:0.1:1787761020.098193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65bc0.
00010000:00000010:3.1:1787761020.562200:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310e00.
00010000:00000010:3.1:1787761020.562214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b8800.
00000080:00000010:2.1:1787761020.608175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345dc0.
00000080:00000010:2.1:1787761020.608190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e8b65bc0.
00010000:00000010:3.1:1787761021.101583:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c700.
00010000:00000010:3.1:1787761021.101598:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1b000.
00000080:00000010:3.1:1787761021.146866:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63459c0.
00000080:00000010:3.1:1787761021.146890:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf380.
00010000:00000010:3.1:1787761021.698672:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4000.
00010000:00000010:3.1:1787761021.698688:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877ea00.
00000080:00000010:2.1:1787761021.720210:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63456a0.
00000080:00000010:2.1:1787761021.720224:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63459c0.
00010000:00000010:0.1:1787761022.043185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646800.
00010000:00000010:0.1:1787761022.043199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1a800.
00000080:00000010:0.1:1787761022.062286:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf1c0.
00000080:00000010:0.1:1787761022.062311:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acfaa0.
00010000:00000010:2.1:1787761022.433187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c300.
00010000:00000010:2.1:1787761022.433203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cf600.
00000080:00000010:0.1:1787761022.490896:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca660.
00000080:00000010:0.1:1787761022.490913:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca960.
00010000:00000010:0.1:1787761023.026309:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c800.
00010000:00000010:0.1:1787761023.026324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b8800.
00000080:00000010:1.1:1787761023.083461:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf1c0.
00000080:00000010:1.1:1787761023.083476:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0cae80.
00010000:00000010:3.1:1787761023.658190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870cc00.
00010000:00000010:3.1:1787761023.658213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87ce000.
00000080:00000010:0.1:1787761023.726185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345f60.
00000080:00000010:0.1:1787761023.726205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf1e0.
00010000:00000010:2.1:1787761024.253408:0:257570:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310500.
00010000:00000010:2.1:1787761024.253423:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b8600.
00000080:00000010:0.1:1787761024.292211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0ed5240.
00000080:00000010:0.1:1787761024.292226:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca660.
00010000:00000010:0.1:1787761024.773189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646500.
00010000:00000010:0.1:1787761024.773204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1b000.
00000080:00000010:0.1:1787761024.845160:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf380.
00000080:00000010:0.1:1787761024.845180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf1c0.
00010000:00000010:2.1:1787761025.243188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646500.
00010000:00000010:2.1:1787761025.243205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865ce00.
00000080:00000010:0.1:1787761025.294189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0ed5240.
00000080:00000010:0.1:1787761025.294207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca600.
00010000:00000010:3.1:1787761025.774234:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c800.
00010000:00000010:3.1:1787761025.774246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1b000.
00000080:00000010:0.1:1787761025.815181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63454c0.
00000080:00000010:0.1:1787761025.815199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf1c0.
00010000:00000010:2.1:1787761026.080494:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c700.
00010000:00000010:2.1:1787761026.080509:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cec00.
00000080:00000010:2.1:1787761026.097031:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0ed5240.
00000080:00000010:2.1:1787761026.097045:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca1c0.
00010000:00000010:3.1:1787761026.570211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646500.
00010000:00000010:3.1:1787761026.570224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1be00.
00000080:00000010:0.1:1787761026.689485:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345740.
00000080:00000010:0.1:1787761026.689506:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf380.
00010000:00000010:1.1:1787761027.149171:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310500.
00010000:00000010:1.1:1787761027.149180:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865d200.
00000080:00000010:1.1:1787761027.165176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca8e0.
00000080:00000010:1.1:1787761027.165188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0ed5c60.
00010000:00000010:1.1:1787761027.845193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c900.
00010000:00000010:1.1:1787761027.845207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cea00.
00000080:00000010:1.1:1787761027.911200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca720.
00000080:00000010:1.1:1787761027.911213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca8e0.
00010000:00000010:2.1:1787761028.607493:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4a00.
00010000:00000010:2.1:1787761028.607507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2400.
00000080:00000010:0.1:1787761028.634639:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae9a0.
00000080:00000010:0.1:1787761028.634655:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63453a0.
00010000:00000010:0.1:1787761029.207363:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310400.
00010000:00000010:0.1:1787761029.207380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865de00.
00000080:00000010:0.1:1787761029.273178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf9e0.
00000080:00000010:0.1:1787761029.273194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aeb60.
00010000:00000010:3.1:1787761029.727522:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce700.
00010000:00000010:3.1:1787761029.727561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1b000.
00000080:00000010:3.1:1787761029.827588:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63450c0.
00000080:00000010:3.1:1787761029.831111:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8acf9e0.
00010000:00000010:2.1:1787761030.357532:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310100.
00010000:00000010:2.1:1787761030.357547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87ce200.
00000080:00000010:0.1:1787761030.413189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aef60.
00000080:00000010:0.1:1787761030.413200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca720.
00010000:00000010:3.1:1787761030.869046:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce600.
00010000:00000010:3.1:1787761030.869060:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865da00.
00000080:00000010:3.1:1787761030.894383:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63450c0.
00000080:00000010:3.1:1787761030.894397:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae640.
00010000:00000010:3.1:1787761031.380195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce800.
00010000:00000010:3.1:1787761031.380210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8785c00.
00000080:00000010:3.1:1787761031.396605:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae900.
00000080:00000010:3.1:1787761031.396620:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca660.
00010000:00000010:3.1:1787761032.136465:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870cf00.
00010000:00000010:3.1:1787761032.136479:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cf800.
00000080:00000010:2.1:1787761032.155968:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63450c0.
00000080:00000010:2.1:1787761032.156184:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca660.
00010000:00000010:3.1:1787761032.876875:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c700.
00010000:00000010:3.1:1787761032.876889:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cf400.
00000080:00000010:0.1:1787761032.917617:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345be0.
00000080:00000010:0.1:1787761032.917636:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cabe0.
00010000:00000010:2.1:1787761033.274237:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4300.
00010000:00000010:2.1:1787761033.274252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865c400.
00000080:00000010:2.1:1787761033.340190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae280.
00000080:00000010:2.1:1787761033.340203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae780.
00010000:00000010:2.1:1787761033.694747:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011cee00.
00010000:00000010:2.1:1787761033.694761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8785e00.
00000080:00000010:3.1:1787761033.737179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae320.
00000080:00000010:3.1:1787761033.737198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca660.
00010000:00000010:2.1:1787761034.260623:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce500.
00010000:00000010:2.1:1787761034.260637:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8784a00.
00000080:00000010:2.1:1787761034.298857:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae460.
00000080:00000010:2.1:1787761034.298873:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6f05a80.
00010000:00000010:3.1:1787761034.803187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce900.
00010000:00000010:3.1:1787761034.803201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8785800.
00000080:00000010:3.1:1787761034.832755:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63451c0.
00000080:00000010:3.1:1787761034.832785:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dce60.
00010000:00000010:3.1:1787761035.376787:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4b00.
00010000:00000010:3.1:1787761035.376803:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2e00.
00000080:00000010:3.1:1787761035.396866:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63459a0.
00000080:00000010:3.1:1787761035.396883:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345620.
00010000:00000010:3.1:1787761035.986369:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c700.
00010000:00000010:3.1:1787761035.986380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cfa00.
00000080:00000010:2.1:1787761036.016228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345620.
00000080:00000010:2.1:1787761036.016246:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca6e0.
00010000:00000010:0.1:1787761036.514233:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce100.
00010000:00000010:0.1:1787761036.514246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8785400.
00000080:00000010:0.1:1787761036.567243:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303e00.
00000080:00000010:0.1:1787761036.567258:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c302b5c0.
00010000:00000010:2.1:1787761037.198228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4c00.
00010000:00000010:2.1:1787761037.198242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2e00.
00000080:00000010:2.1:1787761037.217521:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aee20.
00000080:00000010:2.1:1787761037.217535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63459a0.
00010000:00000010:3.1:1787761037.800183:0:257570:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c700.
00010000:00000010:3.1:1787761037.800190:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865d200.
00000080:00000010:2.1:1787761037.812181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345d00.
00000080:00000010:2.1:1787761037.812198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aee00.
00010000:00000010:2.1:1787761038.354188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce200.
00010000:00000010:2.1:1787761038.354214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8784e00.
00000080:00000010:2.1:1787761038.376238:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aee00.
00000080:00000010:2.1:1787761038.376299:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d93038e0.
00010000:00000010:1.1:1787761038.651188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce200.
00010000:00000010:1.1:1787761038.651203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f3a00.
00000080:00000010:1.1:1787761038.689172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caa80.
00000080:00000010:1.1:1787761038.689183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345f00.
00010000:00000010:2.1:1787761039.077742:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870bb00.
00010000:00000010:2.1:1787761039.077757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87ce800.
00000080:00000010:2.1:1787761039.123062:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae2e0.
00000080:00000010:2.1:1787761039.123091:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caa80.
00010000:00000010:3.1:1787761039.742292:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce500.
00010000:00000010:3.1:1787761039.742386:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8785000.
00000080:00000010:2.1:1787761039.797178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345c60.
00000080:00000010:2.1:1787761039.797205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303b60.
00010000:00000010:1.1:1787761040.407396:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce800.
00010000:00000010:1.1:1787761040.407411:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8784600.
00000080:00000010:1.1:1787761040.458334:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca660.
00000080:00000010:1.1:1787761040.458361:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303b60.
00010000:00000010:3.1:1787761041.214963:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce600.
00010000:00000010:3.1:1787761041.214973:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f3000.
00000080:00000010:0.1:1787761041.267982:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345660.
00000080:00000010:0.1:1787761041.267993:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345320.
00010000:00000010:1.1:1787761041.655182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870be00.
00010000:00000010:1.1:1787761041.655196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cfa00.
00000080:00000010:1.1:1787761041.728185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cae60.
00000080:00000010:1.1:1787761041.728202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca2a0.
00010000:00000010:2.1:1787761042.181191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4500.
00010000:00000010:2.1:1787761042.181212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f3000.
00000080:00000010:1.1:1787761042.239040:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae8e0.
00000080:00000010:1.1:1787761042.239054:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345c80.
00010000:00000010:2.1:1787761042.714992:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4d00.
00010000:00000010:2.1:1787761042.715007:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865ce00.
00000080:00000010:2.1:1787761042.715011:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae6a0.
00000080:00000010:2.1:1787761042.715019:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae740.
00010000:00000010:1.1:1787761043.052238:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce600.
00010000:00000010:1.1:1787761043.052248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8785200.
00000080:00000010:2.1:1787761043.065801:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca2a0.
00000080:00000010:2.1:1787761043.065816:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303b60.
00010000:00000010:0.1:1787761043.448676:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b500.
00010000:00000010:0.1:1787761043.448685:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cfa00.
00000080:00000010:0.1:1787761043.456170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303b60.
00000080:00000010:0.1:1787761043.456186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cae60.
00010000:00000010:3.1:1787761043.935333:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce500.
00010000:00000010:3.1:1787761043.935352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8784000.
00000080:00000010:0.1:1787761043.990719:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345600.
00000080:00000010:0.1:1787761043.990739:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303b60.
00010000:00000010:2.1:1787761044.345195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310d00.
00010000:00000010:2.1:1787761044.345210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2600.
00000080:00000010:2.1:1787761044.410910:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae220.
00000080:00000010:2.1:1787761044.410931:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345d80.
00010000:00000010:2.1:1787761044.755301:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce500.
00010000:00000010:2.1:1787761044.755316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8784000.
00000080:00000010:3.1:1787761044.807179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aeb20.
00000080:00000010:3.1:1787761044.807194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303b60.
00010000:00000010:1.1:1787761045.205358:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310e00.
00010000:00000010:1.1:1787761045.205373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53aac00.
00000080:00000010:1.1:1787761045.263184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca580.
00000080:00000010:1.1:1787761045.263201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aee80.
00010000:00000010:0.1:1787761045.765186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce500.
00010000:00000010:0.1:1787761045.765201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8784600.
00000080:00000010:0.1:1787761045.827178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5d40.
00000080:00000010:0.1:1787761045.827192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0d6a140.
00010000:00000010:1.1:1787761046.453194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870bd00.
00010000:00000010:1.1:1787761046.453206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cee00.
00000080:00000010:1.1:1787761046.494191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caf20.
00000080:00000010:1.1:1787761046.494208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca580.
00010000:00000010:1.1:1787761046.746520:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85ba00.
00010000:00000010:1.1:1787761046.746534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cf400.
00000080:00000010:1.1:1787761046.775216:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caf20.
00000080:00000010:1.1:1787761046.775229:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca580.
00010000:00000010:0.1:1787761047.065189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310b00.
00010000:00000010:0.1:1787761047.065204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53ab000.
00000080:00000010:0.1:1787761047.132274:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5d40.
00000080:00000010:0.1:1787761047.132296:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aeaa0.
00010000:00000010:0.1:1787761047.538179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310700.
00010000:00000010:0.1:1787761047.538189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53ab400.
00000080:00000010:0.1:1787761047.560178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5d40.
00000080:00000010:0.1:1787761047.560193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae980.
00010000:00000010:2.1:1787761048.025474:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4400.
00010000:00000010:2.1:1787761048.025489:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2000.
00000080:00000010:0.1:1787761048.038382:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae260.
00000080:00000010:0.1:1787761048.038398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345460.
00010000:00000010:3.1:1787761048.533305:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b300.
00010000:00000010:3.1:1787761048.533321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8785000.
00000080:00000010:3.1:1787761048.607107:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345700.
00000080:00000010:3.1:1787761048.607121:0:32:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5040.
00010000:00000010:2.1:1787761049.090382:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b100.
00010000:00000010:2.1:1787761049.090397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87ce200.
00000080:00000010:0.1:1787761049.152816:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae360.
00000080:00000010:0.1:1787761049.152836:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caf20.
00010000:00000010:3.1:1787761049.691447:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85bb00.
00010000:00000010:3.1:1787761049.691463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8785800.
00000080:00000010:3.1:1787761049.721173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345700.
00000080:00000010:3.1:1787761049.721188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5cc0.
00010000:00000010:3.1:1787761050.343338:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85bc00.
00010000:00000010:3.1:1787761050.343360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53abe00.
00000080:00000010:3.1:1787761050.378008:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345d80.
00000080:00000010:3.1:1787761050.378684:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345460.
00010000:00000010:2.1:1787761051.028188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b000.
00010000:00000010:2.1:1787761051.028203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cec00.
00000080:00000010:0.1:1787761051.098201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae580.
00000080:00000010:0.1:1787761051.098219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cad00.
00010000:00000010:2.1:1787761051.464222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4500.
00010000:00000010:2.1:1787761051.464239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2c00.
00000080:00000010:0.1:1787761051.525180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae820.
00000080:00000010:0.1:1787761051.525201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345d80.
00010000:00000010:0.1:1787761052.064061:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310b00.
00010000:00000010:0.1:1787761052.064072:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53ab800.
00000080:00000010:1.1:1787761052.123341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac57c0.
00000080:00000010:1.1:1787761052.123357:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae100.
00010000:00000010:0.1:1787761052.617551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310b00.
00010000:00000010:0.1:1787761052.617566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53aa800.
00000080:00000010:0.1:1787761052.686110:0:1051:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5f20.
00000080:00000010:0.1:1787761052.686125:0:1051:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aed20.
00010000:00000010:0.1:1787761053.101294:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85bf00.
00010000:00000010:0.1:1787761053.101308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8784a00.
00000080:00000010:0.1:1787761053.162194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5200.
00000080:00000010:0.1:1787761053.162213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5f20.
00010000:00000010:0.1:1787761053.822455:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870ba00.
00010000:00000010:0.1:1787761053.822471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cf600.
00000080:00000010:0.1:1787761053.871539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5f20.
00000080:00000010:0.1:1787761053.871564:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caaa0.
00010000:00000010:2.1:1787761054.301228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772400.
00010000:00000010:2.1:1787761054.301241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53aa800.
00000080:00000010:2.1:1787761054.326192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae940.
00000080:00000010:2.1:1787761054.326205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae240.
00010000:00000010:2.1:1787761054.840332:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b500.
00010000:00000010:2.1:1787761054.840349:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8784a00.
00000080:00000010:0.1:1787761054.859393:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aede0.
00000080:00000010:0.1:1787761054.859413:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5f20.
00010000:00000010:2.1:1787761055.188416:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4600.
00010000:00000010:2.1:1787761055.188433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2c00.
00000080:00000010:0.1:1787761055.257236:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aea60.
00000080:00000010:0.1:1787761055.257254:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345320.
00010000:00000010:2.1:1787761055.807515:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772400.
00010000:00000010:2.1:1787761055.807533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2600.
00000080:00000010:0.1:1787761055.839323:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aed40.
00000080:00000010:0.1:1787761055.839343:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345c60.
00010000:00000010:0.1:1787761056.396326:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b900.
00010000:00000010:0.1:1787761056.396340:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8785c00.
00000080:00000010:0.1:1787761056.477349:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5940.
00000080:00000010:0.1:1787761056.477368:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5200.
00010000:00000010:3.1:1787761056.846203:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4a00.
00010000:00000010:3.1:1787761056.846218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f3a00.
00000080:00000010:3.1:1787761056.872160:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345d00.
00000080:00000010:3.1:1787761056.872174:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345f00.
00010000:00000010:2.1:1787761057.128188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4600.
00010000:00000010:2.1:1787761057.128204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2e00.
00000080:00000010:0.1:1787761057.171178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae820.
00000080:00000010:0.1:1787761057.171197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345d00.
00010000:00000010:3.1:1787761057.474176:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b300.
00010000:00000010:3.1:1787761057.474192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cf200.
00000080:00000010:3.1:1787761057.496181:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63459a0.
00000080:00000010:3.1:1787761057.496195:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca820.
00010000:00000010:2.1:1787761058.092716:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b800.
00010000:00000010:2.1:1787761058.092732:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cf600.
00000080:00000010:0.1:1787761058.141164:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae620.
00000080:00000010:0.1:1787761058.141176:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5200.
00010000:00000010:2.1:1787761058.783186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b100.
00010000:00000010:2.1:1787761058.783203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8784a00.
00000080:00000010:0.1:1787761058.856177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae780.
00000080:00000010:0.1:1787761058.856187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5940.
00010000:00000010:3.1:1787761059.237178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772400.
00010000:00000010:3.1:1787761059.237195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53aa000.
00000080:00000010:3.1:1787761059.284179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63459a0.
00000080:00000010:3.1:1787761059.284198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae280.
00010000:00000010:2.1:1787761059.729855:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b000.
00010000:00000010:2.1:1787761059.729869:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2400.
00000080:00000010:0.1:1787761059.760188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae2c0.
00000080:00000010:0.1:1787761059.760205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63459a0.
00010000:00000010:0.1:1787761060.176639:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2200.
00010000:00000010:0.1:1787761060.176654:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2c00.
00000080:00000010:0.1:1787761060.199040:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5940.
00000080:00000010:0.1:1787761060.199056:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63450c0.
00010000:00000010:3.1:1787761060.867270:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b300.
00010000:00000010:3.1:1787761060.867280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cec00.
00000080:00000010:3.1:1787761060.867298:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63453a0.
00000080:00000010:3.1:1787761060.867307:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caca0.
00010000:00000010:2.1:1787761061.378441:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2000.
00010000:00000010:2.1:1787761061.378455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53abc00.
00000080:00000010:3.1:1787761061.401496:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae6c0.
00000080:00000010:3.1:1787761061.434755:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aee40.
00010000:00000010:2.1:1787761061.915277:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2a00.
00010000:00000010:2.1:1787761061.915291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2c00.
00000080:00000010:0.1:1787761061.928169:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aeaa0.
00000080:00000010:0.1:1787761061.928183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63454c0.
00010000:00000010:3.1:1787761062.539178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2300.
00010000:00000010:3.1:1787761062.539192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8790e00.
00000080:00000010:3.1:1787761062.626188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63459c0.
00000080:00000010:3.1:1787761062.626204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345f60.
00010000:00000010:2.1:1787761063.147184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b000.
00010000:00000010:2.1:1787761063.147199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8785000.
00000080:00000010:0.1:1787761063.225455:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aebe0.
00000080:00000010:0.1:1787761063.225476:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac50a0.
00010000:00000010:1.1:1787761063.643178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b100.
00010000:00000010:1.1:1787761063.643192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53ab800.
00000080:00000010:1.1:1787761063.668180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cade0.
00000080:00000010:1.1:1787761063.668196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae680.
00010000:00000010:3.1:1787761064.021779:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2800.
00010000:00000010:3.1:1787761064.021788:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791a00.
00000080:00000010:3.1:1787761064.060188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63456a0.
00000080:00000010:3.1:1787761064.060204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63459c0.
00010000:00000010:3.1:1787761064.466470:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b500.
00010000:00000010:3.1:1787761064.466480:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cec00.
00000080:00000010:3.1:1787761064.509215:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63459c0.
00000080:00000010:3.1:1787761064.509228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cade0.
00010000:00000010:3.1:1787761065.089196:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b500.
00010000:00000010:3.1:1787761065.089212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53abe00.
00000080:00000010:3.1:1787761065.111185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63459c0.
00000080:00000010:3.1:1787761065.111205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae8e0.
00010000:00000010:0.1:1787761065.374025:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85bb00.
00010000:00000010:0.1:1787761065.374039:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8785000.
00000080:00000010:0.1:1787761065.387170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac50e0.
00000080:00000010:0.1:1787761065.387184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5640.
00010000:00000010:2.1:1787761065.736297:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870bf00.
00010000:00000010:2.1:1787761065.736306:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cf600.
00000080:00000010:0.1:1787761065.795459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae8e0.
00000080:00000010:0.1:1787761065.795470:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5640.
00010000:00000010:2.1:1787761066.168165:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85ba00.
00010000:00000010:2.1:1787761066.168174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8784a00.
00000080:00000010:0.1:1787761066.218541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae220.
00000080:00000010:0.1:1787761066.218553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5640.
00010000:00000010:2.1:1787761066.606895:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b500.
00010000:00000010:2.1:1787761066.606911:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d7200.
00000080:00000010:1.1:1787761066.668818:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aeb20.
00000080:00000010:1.1:1787761066.668831:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5640.
00010000:00000010:2.1:1787761067.174120:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b000.
00010000:00000010:2.1:1787761067.174146:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d7600.
00000080:00000010:0.1:1787761067.236681:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae700.
00000080:00000010:0.1:1787761067.236702:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5260.
00010000:00000010:2.1:1787761067.625277:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870bf00.
00010000:00000010:2.1:1787761067.625292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cf600.
00000080:00000010:1.1:1787761067.689169:0:1051:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae600.
00000080:00000010:1.1:1787761067.689184:0:1051:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca0a0.
00010000:00000010:2.1:1787761068.037198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b500.
00010000:00000010:2.1:1787761068.037214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d6000.
00000080:00000010:1.1:1787761068.077176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aee80.
00000080:00000010:1.1:1787761068.077190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5260.
00010000:00000010:0.1:1787761068.378238:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53e00.
00010000:00000010:0.1:1787761068.378253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d7e00.
00000080:00000010:0.1:1787761068.421174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5980.
00000080:00000010:0.1:1787761068.421190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac53c0.
00010000:00000010:0.1:1787761068.897243:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b500.
00010000:00000010:0.1:1787761068.897261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cec00.
00000080:00000010:0.1:1787761068.921507:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac53c0.
00000080:00000010:0.1:1787761068.921523:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca0a0.
00010000:00000010:2.1:1787761069.378819:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b300.
00010000:00000010:2.1:1787761069.378833:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d6c00.
00000080:00000010:0.1:1787761069.437478:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae680.
00000080:00000010:0.1:1787761069.453251:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac53c0.
00010000:00000010:2.1:1787761069.870188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772100.
00010000:00000010:2.1:1787761069.870200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791000.
00000080:00000010:3.1:1787761069.922643:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae2e0.
00000080:00000010:3.1:1787761069.922659:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63459c0.
00010000:00000010:3.1:1787761070.402290:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2e00.
00010000:00000010:3.1:1787761070.402315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8790600.
00000080:00000010:0.1:1787761070.470164:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63450e0.
00000080:00000010:0.1:1787761070.470177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345dc0.
00010000:00000010:0.1:1787761070.886658:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b300.
00010000:00000010:0.1:1787761070.886675:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cf600.
00000080:00000010:0.1:1787761070.935200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5d60.
00000080:00000010:0.1:1787761070.935215:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cac80.
00010000:00000010:3.1:1787761071.440186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53d00.
00010000:00000010:3.1:1787761071.440200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cec00.
00000080:00000010:1.1:1787761071.476108:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aea20.
00000080:00000010:1.1:1787761071.476130:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca7e0.
00010000:00000010:2.1:1787761071.985186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53400.
00010000:00000010:2.1:1787761071.985201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d7800.
00000080:00000010:3.1:1787761072.019287:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae860.
00000080:00000010:3.1:1787761072.019301:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5d60.
00010000:00000010:3.1:1787761072.645956:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870bf00.
00010000:00000010:3.1:1787761072.645974:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cec00.
00000080:00000010:3.1:1787761072.677190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63458e0.
00000080:00000010:3.1:1787761072.677211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca7e0.
00010000:00000010:2.1:1787761073.031276:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b300.
00010000:00000010:2.1:1787761073.031293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ac600.
00000080:00000010:2.1:1787761073.057399:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae040.
00000080:00000010:2.1:1787761073.057416:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca740.
00010000:00000010:2.1:1787761073.349182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772200.
00010000:00000010:2.1:1787761073.349194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2c00.
00000080:00000010:2.1:1787761073.372295:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae340.
00000080:00000010:2.1:1787761073.372308:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae440.
00010000:00000010:0.1:1787761073.725971:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2f00.
00010000:00000010:0.1:1787761073.725983:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791600.
00000080:00000010:0.1:1787761073.771874:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5300.
00000080:00000010:0.1:1787761073.771886:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345940.
00010000:00000010:0.1:1787761074.289286:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53000.
00010000:00000010:0.1:1787761074.289301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d6e00.
00000080:00000010:0.1:1787761074.346370:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5100.
00000080:00000010:0.1:1787761074.346387:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5300.
00010000:00000010:2.1:1787761074.773604:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53000.
00010000:00000010:2.1:1787761074.773619:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2c00.
00000080:00000010:0.1:1787761074.810173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae140.
00000080:00000010:0.1:1787761074.810186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae780.
00010000:00000010:3.1:1787761075.321194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53a00.
00010000:00000010:3.1:1787761075.321208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d7000.
00000080:00000010:0.1:1787761075.339192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345160.
00000080:00000010:0.1:1787761075.339220:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5100.
00010000:00000010:2.1:1787761075.684493:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772700.
00010000:00000010:2.1:1787761075.684502:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791c00.
00000080:00000010:0.1:1787761075.744207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae100.
00000080:00000010:0.1:1787761075.744220:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345dc0.
00010000:00000010:2.1:1787761076.163440:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772a00.
00010000:00000010:2.1:1787761076.163457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbff400.
00000080:00000010:2.1:1787761076.174201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aeb40.
00000080:00000010:2.1:1787761076.174219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae3c0.
00010000:00000010:0.1:1787761076.563904:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53600.
00010000:00000010:0.1:1787761076.563918:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbff000.
00000080:00000010:0.1:1787761076.584617:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5100.
00000080:00000010:0.1:1787761076.584635:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aeb60.
00010000:00000010:1.1:1787761076.992433:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772300.
00010000:00000010:1.1:1787761076.992440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbffa00.
00000080:00000010:1.1:1787761077.031265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca900.
00000080:00000010:1.1:1787761077.031284:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae400.
00010000:00000010:3.1:1787761077.636184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53300.
00010000:00000010:3.1:1787761077.636198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d6600.
00000080:00000010:3.1:1787761077.654651:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63450e0.
00000080:00000010:3.1:1787761077.654665:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5960.
00010000:00000010:0.1:1787761078.109317:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53b00.
00010000:00000010:0.1:1787761078.109333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d6600.
00000080:00000010:0.1:1787761078.160732:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5c40.
00000080:00000010:0.1:1787761078.160743:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5120.
00010000:00000010:3.1:1787761078.601461:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53c00.
00010000:00000010:3.1:1787761078.601485:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d7000.
00000080:00000010:2.1:1787761078.617192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63450e0.
00000080:00000010:2.1:1787761078.617206:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5c40.
00010000:00000010:3.1:1787761079.101410:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772b00.
00010000:00000010:3.1:1787761079.101425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbff200.
00000080:00000010:2.1:1787761079.190699:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63454c0.
00000080:00000010:2.1:1787761079.190717:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae400.
00010000:00000010:2.1:1787761079.622230:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53c00.
00010000:00000010:2.1:1787761079.622246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbfec00.
00000080:00000010:2.1:1787761079.701716:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae640.
00000080:00000010:2.1:1787761079.701734:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae400.
00010000:00000010:1.1:1787761080.256170:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772f00.
00010000:00000010:1.1:1787761080.256187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791e00.
00000080:00000010:1.1:1787761080.324175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cadc0.
00000080:00000010:1.1:1787761080.324191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae640.
00010000:00000010:3.1:1787761080.866517:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53a00.
00010000:00000010:3.1:1787761080.866540:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d6c00.
00000080:00000010:2.1:1787761080.913722:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63450c0.
00000080:00000010:2.1:1787761080.913735:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac51a0.
00010000:00000010:2.1:1787761081.294772:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53400.
00010000:00000010:2.1:1787761081.294787:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d7600.
00000080:00000010:0.1:1787761081.367217:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae140.
00000080:00000010:0.1:1787761081.367230:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5d20.
00010000:00000010:0.1:1787761081.949191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b800.
00010000:00000010:0.1:1787761081.949207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ac000.
00000080:00000010:0.1:1787761081.990175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5d20.
00000080:00000010:0.1:1787761081.990191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cafe0.
00010000:00000010:1.1:1787761082.482292:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128bc00.
00010000:00000010:1.1:1787761082.482306:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84adc00.
00000080:00000010:2.1:1787761082.517523:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca980.
00000080:00000010:2.1:1787761082.517545:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cab00.
00010000:00000010:3.1:1787761083.244240:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128bc00.
00010000:00000010:3.1:1787761083.244254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84adc00.
00000080:00000010:3.1:1787761083.314659:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63459a0.
00000080:00000010:3.1:1787761083.314678:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cad80.
00010000:00000010:0.1:1787761084.047289:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53a00.
00010000:00000010:0.1:1787761084.047303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d6c00.
00000080:00000010:0.1:1787761084.064662:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5c00.
00000080:00000010:0.1:1787761084.064679:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5b40.
00010000:00000010:2.1:1787761084.650195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128be00.
00010000:00000010:2.1:1787761084.650221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbff200.
00000080:00000010:0.1:1787761084.714322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae020.
00000080:00000010:0.1:1787761084.718353:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae8a0.
00010000:00000010:0.1:1787761085.062201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53c00.
00010000:00000010:0.1:1787761085.062222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d7000.
00000080:00000010:0.1:1787761085.159190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5ae0.
00000080:00000010:0.1:1787761085.159207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5c00.
00010000:00000010:2.1:1787761085.830526:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2400.
00010000:00000010:2.1:1787761085.830539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8790c00.
00000080:00000010:0.1:1787761085.855181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae8c0.
00000080:00000010:0.1:1787761085.855195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345d00.
00010000:00000010:1.1:1787761086.306382:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797ad00.
00010000:00000010:1.1:1787761086.306390:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d7000.
00000080:00000010:1.1:1787761086.348307:0:1051:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca520.
00000080:00000010:1.1:1787761086.355185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5c00.
00010000:00000010:2.1:1787761086.659171:0:257570:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2400.
00010000:00000010:2.1:1787761086.659182:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791800.
00000080:00000010:0.1:1787761086.698174:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae8e0.
00000080:00000010:0.1:1787761086.698184:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345c60.
00010000:00000010:0.1:1787761086.965186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797a400.
00010000:00000010:0.1:1787761086.965201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d6c00.
00000080:00000010:0.1:1787761087.027212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac51c0.
00000080:00000010:0.1:1787761087.027225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5de0.
00010000:00000010:3.1:1787761087.467196:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797a400.
00010000:00000010:3.1:1787761087.467209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbffa00.
00000080:00000010:3.1:1787761087.490314:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345320.
00000080:00000010:3.1:1787761087.490326:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aeec0.
00010000:00000010:2.1:1787761087.937178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797a100.
00010000:00000010:2.1:1787761087.937193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791400.
00000080:00000010:3.1:1787761087.958575:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aeb80.
00000080:00000010:3.1:1787761087.958588:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345d80.
00010000:00000010:3.1:1787761088.238295:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772f00.
00010000:00000010:3.1:1787761088.238308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbfe800.
00000080:00000010:0.1:1787761088.293170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345d80.
00000080:00000010:0.1:1787761088.293180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae0a0.
00010000:00000010:0.1:1787761088.728713:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128bf00.
00010000:00000010:0.1:1787761088.728721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84aca00.
00000080:00000010:0.1:1787761088.781039:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5520.
00000080:00000010:0.1:1787761088.781054:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac51c0.
00010000:00000010:1.1:1787761089.477195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797a000.
00010000:00000010:1.1:1787761089.477206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791800.
00000080:00000010:1.1:1787761089.526210:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca520.
00000080:00000010:1.1:1787761089.526223:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345c60.
00010000:00000010:3.1:1787761089.854869:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797aa00.
00010000:00000010:3.1:1787761089.854884:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b1e00.
00000080:00000010:2.1:1787761089.896044:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345d00.
00000080:00000010:2.1:1787761089.896060:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5520.
00010000:00000010:1.1:1787761090.307960:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772400.
00010000:00000010:1.1:1787761090.307976:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbff800.
00000080:00000010:1.1:1787761090.322939:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca520.
00000080:00000010:1.1:1787761090.322953:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae8a0.
00010000:00000010:1.1:1787761090.950401:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797a300.
00010000:00000010:1.1:1787761090.950414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b0c00.
00000080:00000010:1.1:1787761090.981171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca520.
00000080:00000010:1.1:1787761090.981184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5560.
00010000:00000010:0.1:1787761091.696246:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772f00.
00010000:00000010:0.1:1787761091.696259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbff800.
00000080:00000010:0.1:1787761091.714376:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac54a0.
00000080:00000010:0.1:1787761091.714402:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aeaa0.
00010000:00000010:1.1:1787761092.464334:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128bd00.
00010000:00000010:1.1:1787761092.464341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ad800.
00000080:00000010:1.1:1787761092.484899:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cab00.
00000080:00000010:1.1:1787761092.484909:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cad80.
00010000:00000010:0.1:1787761093.210227:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797a800.
00010000:00000010:0.1:1787761093.210250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b0200.
00000080:00000010:0.1:1787761093.345260:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5020.
00000080:00000010:0.1:1787761093.345276:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac54a0.
00010000:00000010:3.1:1787761093.913542:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2700.
00010000:00000010:3.1:1787761093.913556:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b0200.
00000080:00000010:3.1:1787761093.933078:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63454c0.
00000080:00000010:3.1:1787761093.949614:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac54a0.
00010000:00000010:1.1:1787761094.394185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b400.
00010000:00000010:1.1:1787761094.394198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ad400.
00000080:00000010:1.1:1787761094.420173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cafe0.
00000080:00000010:1.1:1787761094.420184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca980.
00010000:00000010:3.1:1787761095.042191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797ab00.
00010000:00000010:3.1:1787761095.042205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b1800.
00000080:00000010:2.1:1787761095.086819:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63454c0.
00000080:00000010:2.1:1787761095.086837:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5020.
00010000:00000010:3.1:1787761095.600300:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b100.
00010000:00000010:3.1:1787761095.600315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ad400.
00000080:00000010:3.1:1787761095.625113:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63450e0.
00000080:00000010:3.1:1787761095.625164:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca980.
00010000:00000010:0.1:1787761096.270204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2f00.
00010000:00000010:0.1:1787761096.270219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791c00.
00000080:00000010:1.1:1787761096.315253:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5aa0.
00000080:00000010:1.1:1787761096.315266:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63450e0.
00010000:00000010:2.1:1787761096.866300:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797ac00.
00010000:00000010:2.1:1787761096.866316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2c00.
00000080:00000010:2.1:1787761096.939350:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae780.
00000080:00000010:2.1:1787761096.939362:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae4c0.
00010000:00000010:0.1:1787761097.579585:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797af00.
00010000:00000010:0.1:1787761097.579603:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b1400.
00000080:00000010:0.1:1787761097.619852:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5060.
00000080:00000010:0.1:1787761097.619869:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5a00.
00010000:00000010:2.1:1787761098.517354:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797a500.
00010000:00000010:2.1:1787761098.517369:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b1400.
00000080:00000010:3.1:1787761098.538033:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aebc0.
00000080:00000010:3.1:1787761098.538048:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5a00.
00010000:00000010:3.1:1787761099.248301:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b200.
00010000:00000010:3.1:1787761099.248317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ad000.
00000080:00000010:3.1:1787761099.316197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63450e0.
00000080:00000010:3.1:1787761099.316212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca7e0.
00010000:00000010:0.1:1787761100.097302:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797ab00.
00010000:00000010:0.1:1787761100.097316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b1000.
00000080:00000010:2.1:1787761100.126058:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5ee0.
00000080:00000010:2.1:1787761100.126072:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5f00.
00010000:00000010:3.1:1787761100.947316:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b700.
00010000:00000010:3.1:1787761100.947327:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ad000.
00000080:00000010:0.1:1787761101.010417:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63454c0.
00000080:00000010:0.1:1787761101.010434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca800.
00010000:00000010:3.1:1787761101.624202:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b600.
00010000:00000010:3.1:1787761101.624216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84aca00.
00000080:00000010:0.1:1787761101.699544:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345d00.
00000080:00000010:0.1:1787761101.699559:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cade0.
00010000:00000010:0.1:1787761102.609446:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797ab00.
00010000:00000010:0.1:1787761102.609461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7835200.
00000080:00000010:0.1:1787761102.639320:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac52a0.
00000080:00000010:0.1:1787761102.639338:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aebc0.
00010000:00000010:3.1:1787761103.428192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797a300.
00010000:00000010:3.1:1787761103.428216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b1000.
00000080:00000010:0.1:1787761103.476607:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6345d80.
00000080:00000010:0.1:1787761103.476623:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac52a0.
00010000:00000010:0.1:1787761104.266186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b600.
00010000:00000010:0.1:1787761104.266200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84adc00.
00000080:00000010:0.1:1787761104.310970:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac52a0.
00000080:00000010:0.1:1787761104.310988:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca2e0.
00010000:00000010:1.1:1787761105.010190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b200.
00010000:00000010:1.1:1787761105.010208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ac600.
00000080:00000010:1.1:1787761105.078068:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca5c0.
00000080:00000010:1.1:1787761105.078084:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca820.
00010000:00000010:2.1:1787761105.914185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2000.
00010000:00000010:2.1:1787761105.914197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791800.
00000080:00000010:0.1:1787761105.959956:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae280.
00000080:00000010:0.1:1787761105.959974:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63454c0.
00010000:00000010:2.1:1787761106.548806:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2000.
00010000:00000010:2.1:1787761106.548817:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791800.
00000080:00000010:0.1:1787761106.580125:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae500.
00000080:00000010:0.1:1787761106.580157:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c63454c0.
00010000:00000010:3.1:1787761107.449374:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2400.
00010000:00000010:3.1:1787761107.449389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873a400.
00000080:00000010:3.1:1787761107.461824:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4dc0.
00000080:00000010:3.1:1787761107.461842:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4d20.
00010000:00000010:2.1:1787761108.077913:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2400.
00010000:00000010:2.1:1787761108.077929:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873a400.
00000080:00000010:0.1:1787761108.147147:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aece0.
00000080:00000010:0.1:1787761108.147161:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac52a0.
00010000:00000010:1.1:1787761108.714586:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797a400.
00010000:00000010:1.1:1787761108.714604:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b0800.
00000080:00000010:1.1:1787761108.738665:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca820.
00000080:00000010:1.1:1787761108.738678:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac52a0.
00010000:00000010:2.1:1787761109.227871:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b100.
00010000:00000010:2.1:1787761109.227888:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ac600.
00000080:00000010:0.1:1787761109.264158:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae8a0.
00000080:00000010:0.1:1787761109.264173:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca820.
00010000:00000010:0.1:1787761109.580249:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2400.
00010000:00000010:0.1:1787761109.580264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873ae00.
00000080:00000010:1.1:1787761109.602204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5780.
00000080:00000010:1.1:1787761109.602217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4dc0.
00010000:00000010:3.1:1787761109.989540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128be00.
00010000:00000010:3.1:1787761109.989556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ac600.
00000080:00000010:2.1:1787761110.016851:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b45e0.
00000080:00000010:2.1:1787761110.016866:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca5c0.
00010000:00000010:0.1:1787761110.319311:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797a400.
00010000:00000010:0.1:1787761110.319319:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b0600.
00000080:00000010:0.1:1787761110.382195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5700.
00000080:00000010:0.1:1787761110.382215:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5d00.
00010000:00000010:3.1:1787761110.956681:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2400.
00010000:00000010:3.1:1787761110.956695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7834400.
00000080:00000010:2.1:1787761110.984181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4de0.
00000080:00000010:2.1:1787761111.044341:0:293338:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae8e0.
00010000:00000010:0.1:1787761111.316203:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128bc00.
00010000:00000010:0.1:1787761111.316214:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84adc00.
00000080:00000010:0.1:1787761111.348902:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5700.
00000080:00000010:0.1:1787761111.348912:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caaa0.
00010000:00000010:1.1:1787761111.573360:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128bc00.
00010000:00000010:1.1:1787761111.573375:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84aca00.
00000080:00000010:0.1:1787761111.601710:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caf20.
00000080:00000010:0.1:1787761111.601723:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cad00.
00010000:00000010:3.1:1787761112.143385:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945d00.
00010000:00000010:3.1:1787761112.143398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b0800.
00000080:00000010:2.1:1787761112.198192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b43a0.
00000080:00000010:2.1:1787761112.198202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5700.
00010000:00000010:0.1:1787761112.920907:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945400.
00010000:00000010:0.1:1787761112.920915:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b1000.
00000080:00000010:0.1:1787761112.955818:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5000.
00000080:00000010:0.1:1787761112.955837:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5080.
00010000:00000010:0.1:1787761113.577209:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945400.
00010000:00000010:0.1:1787761113.577238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7834800.
00000080:00000010:0.1:1787761113.627358:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5000.
00000080:00000010:0.1:1787761113.627377:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aeb20.
00010000:00000010:2.1:1787761114.586516:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2b00.
00010000:00000010:2.1:1787761114.586525:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873a800.
00000080:00000010:0.1:1787761114.612175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae5c0.
00000080:00000010:0.1:1787761114.612189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4620.
00010000:00000010:1.1:1787761115.635525:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03600.
00010000:00000010:1.1:1787761115.635540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7835800.
00000080:00000010:1.1:1787761115.658486:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca2a0.
00000080:00000010:1.1:1787761115.658502:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae8c0.
00010000:00000010:2.1:1787761116.699815:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945100.
00010000:00000010:2.1:1787761116.699830:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b1800.
00000080:00000010:0.1:1787761116.732970:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae8a0.
00000080:00000010:0.1:1787761116.732984:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac54e0.
00010000:00000010:2.1:1787761117.326198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945000.
00010000:00000010:2.1:1787761117.326213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b0a00.
00000080:00000010:0.1:1787761117.409387:0:1051:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae160.
00000080:00000010:0.1:1787761117.409402:0:1051:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac54e0.
00010000:00000010:3.1:1787761117.989512:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945700.
00010000:00000010:3.1:1787761117.989526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ad000.
00000080:00000010:1.1:1787761118.032371:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4da0.
00000080:00000010:1.1:1787761118.032387:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caa80.
00010000:00000010:0.1:1787761118.608530:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945a00.
00010000:00000010:0.1:1787761118.608539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873b200.
00000080:00000010:0.1:1787761118.669378:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac54e0.
00000080:00000010:0.1:1787761118.669398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4560.
00010000:00000010:3.1:1787761119.108192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945600.
00010000:00000010:3.1:1787761119.108206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b1e00.
00000080:00000010:0.1:1787761119.182251:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4820.
00000080:00000010:0.1:1787761119.182414:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac54e0.
00010000:00000010:2.1:1787761119.733837:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b100.
00010000:00000010:2.1:1787761119.733852:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84adc00.
00000080:00000010:2.1:1787761119.750173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae2c0.
00000080:00000010:2.1:1787761119.750186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca6e0.
00010000:00000010:2.1:1787761120.194449:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03b00.
00010000:00000010:2.1:1787761120.194464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873a000.
00000080:00000010:3.1:1787761120.214488:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae140.
00000080:00000010:3.1:1787761120.214504:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b48a0.
00010000:00000010:3.1:1787761120.823728:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b100.
00010000:00000010:3.1:1787761120.823740:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ac600.
00000080:00000010:3.1:1787761120.823745:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b48a0.
00000080:00000010:3.1:1787761120.823755:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca7a0.
00010000:00000010:2.1:1787761121.262513:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae400.
00010000:00000010:2.1:1787761121.262528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84adc00.
00000080:00000010:2.1:1787761121.289674:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae100.
00000080:00000010:2.1:1787761121.289689:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca600.
00010000:00000010:1.1:1787761121.824203:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945b00.
00010000:00000010:1.1:1787761121.824226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b0200.
00000080:00000010:1.1:1787761121.890190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca960.
00000080:00000010:1.1:1787761121.890203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5620.
00010000:00000010:0.1:1787761122.287193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2e00.
00010000:00000010:0.1:1787761122.287218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7835a00.
00000080:00000010:0.1:1787761122.317177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac56a0.
00000080:00000010:0.1:1787761122.317191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aeb60.
00010000:00000010:3.1:1787761122.840362:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03e00.
00010000:00000010:3.1:1787761122.840375:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873be00.
00000080:00000010:0.1:1787761122.882737:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4b20.
00000080:00000010:0.1:1787761122.882753:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b42a0.
00010000:00000010:3.1:1787761123.448181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945e00.
00010000:00000010:3.1:1787761123.448195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b0800.
00000080:00000010:0.1:1787761123.502681:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4480.
00000080:00000010:0.1:1787761123.502697:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac59a0.
00010000:00000010:2.1:1787761123.987184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2f00.
00010000:00000010:2.1:1787761123.987199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7834800.
00000080:00000010:1.1:1787761124.022211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae620.
00000080:00000010:1.1:1787761124.022224:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aebc0.
00010000:00000010:3.1:1787761124.569045:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945500.
00010000:00000010:3.1:1787761124.569062:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873bc00.
00000080:00000010:0.1:1787761124.630240:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4cc0.
00000080:00000010:0.1:1787761124.630253:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4340.
00010000:00000010:2.1:1787761125.116269:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae200.
00010000:00000010:2.1:1787761125.116293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867c400.
00000080:00000010:2.1:1787761125.146169:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aed60.
00000080:00000010:2.1:1787761125.146181:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caec0.
00010000:00000010:2.1:1787761125.650186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945e00.
00010000:00000010:2.1:1787761125.650210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b1a00.
00000080:00000010:3.1:1787761125.711769:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aece0.
00000080:00000010:3.1:1787761125.711789:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5380.
00010000:00000010:0.1:1787761126.329280:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945b00.
00010000:00000010:0.1:1787761126.329293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7835a00.
00000080:00000010:0.1:1787761126.369679:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5880.
00000080:00000010:0.1:1787761126.369694:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae160.
00010000:00000010:2.1:1787761126.863164:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945300.
00010000:00000010:2.1:1787761126.863181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7835a00.
00000080:00000010:2.1:1787761126.887167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae8a0.
00000080:00000010:2.1:1787761126.887178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aefa0.
00010000:00000010:2.1:1787761127.431186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2d00.
00010000:00000010:2.1:1787761127.431203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7834800.
00000080:00000010:2.1:1787761127.460273:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae540.
00000080:00000010:2.1:1787761127.460292:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae8a0.
00010000:00000010:2.1:1787761128.003265:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945600.
00010000:00000010:2.1:1787761128.003284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873a200.
00000080:00000010:0.1:1787761128.063177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae660.
00000080:00000010:0.1:1787761128.067241:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4ca0.
00010000:00000010:2.1:1787761128.475201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945000.
00010000:00000010:2.1:1787761128.475215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b0200.
00000080:00000010:0.1:1787761128.543594:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae180.
00000080:00000010:0.1:1787761128.543604:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5fe0.
00010000:00000010:0.1:1787761128.836277:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945d00.
00010000:00000010:0.1:1787761128.836292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7834800.
00000080:00000010:0.1:1787761128.886292:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac51e0.
00000080:00000010:0.1:1787761128.886302:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae5c0.
00010000:00000010:3.1:1787761129.367180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945000.
00010000:00000010:3.1:1787761129.367192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843a00.
00000080:00000010:0.1:1787761129.405267:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b40e0.
00000080:00000010:0.1:1787761129.405282:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac51e0.
00010000:00000010:1.1:1787761129.604211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945300.
00010000:00000010:1.1:1787761129.604222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7842800.
00000080:00000010:1.1:1787761129.632181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca780.
00000080:00000010:1.1:1787761129.632197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac51e0.
00010000:00000010:0.1:1787761129.883374:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945300.
00010000:00000010:0.1:1787761129.883388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843200.
00000080:00000010:0.1:1787761129.913182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5440.
00000080:00000010:0.1:1787761129.913196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5b20.
00010000:00000010:2.1:1787761130.306523:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03d00.
00010000:00000010:2.1:1787761130.306539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873a000.
00000080:00000010:0.1:1787761130.350970:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae760.
00000080:00000010:1.1:1787761130.376172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b44c0.
00010000:00000010:3.1:1787761131.251326:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945000.
00010000:00000010:3.1:1787761131.251343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843600.
00000080:00000010:2.1:1787761131.294757:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4160.
00000080:00000010:2.1:1787761131.294778:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5440.
00010000:00000010:2.1:1787761131.986262:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87acb00.
00010000:00000010:2.1:1787761131.986280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7842000.
00000080:00000010:2.1:1787761132.015729:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae220.
00000080:00000010:2.1:1787761132.015742:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac57a0.
00010000:00000010:3.1:1787761132.545188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ace00.
00010000:00000010:3.1:1787761132.545205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843e00.
00000080:00000010:0.1:1787761132.561182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4e40.
00000080:00000010:0.1:1787761132.591165:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac50c0.
00010000:00000010:2.1:1787761133.207148:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2400.
00010000:00000010:2.1:1787761133.207164:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791800.
00000080:00000010:2.1:1787761133.237438:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae540.
00000080:00000010:2.1:1787761133.237450:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae8c0.
00010000:00000010:3.1:1787761133.838378:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae000.
00010000:00000010:3.1:1787761133.838395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7842c00.
00000080:00000010:0.1:1787761133.910105:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4e40.
00000080:00000010:0.1:1787761133.910119:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac50c0.
00010000:00000010:2.1:1787761134.450719:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2100.
00010000:00000010:2.1:1787761134.450734:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791800.
00000080:00000010:2.1:1787761134.476228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aed60.
00000080:00000010:2.1:1787761134.476241:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae260.
00010000:00000010:2.1:1787761135.015352:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae700.
00010000:00000010:2.1:1787761135.015370:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7842a00.
00000080:00000010:0.1:1787761135.038174:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aedc0.
00000080:00000010:0.1:1787761135.038187:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac50c0.
00010000:00000010:3.1:1787761135.544171:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87acd00.
00010000:00000010:3.1:1787761135.544177:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873ee00.
00000080:00000010:3.1:1787761135.636786:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b44c0.
00000080:00000010:3.1:1787761135.636799:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae2c0.
00010000:00000010:0.1:1787761136.193191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac400.
00010000:00000010:0.1:1787761136.193205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7842a00.
00000080:00000010:1.1:1787761136.254389:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5600.
00000080:00000010:1.1:1787761136.254408:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5340.
00010000:00000010:2.1:1787761136.786447:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac100.
00010000:00000010:2.1:1787761136.786462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873e000.
00000080:00000010:0.1:1787761136.830190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae9e0.
00000080:00000010:0.1:1787761136.830203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aebc0.
00010000:00000010:3.1:1787761137.415965:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac000.
00010000:00000010:3.1:1787761137.415982:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7842a00.
00000080:00000010:3.1:1787761137.443305:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b44c0.
00000080:00000010:3.1:1787761137.443320:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5e60.
00010000:00000010:3.1:1787761137.968187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03100.
00010000:00000010:3.1:1787761137.968201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873a200.
00000080:00000010:3.1:1787761137.983340:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4ca0.
00000080:00000010:3.1:1787761137.996332:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b40e0.
00010000:00000010:2.1:1787761138.531183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87aca00.
00010000:00000010:2.1:1787761138.531201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7842200.
00000080:00000010:0.1:1787761138.596287:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aeb60.
00000080:00000010:0.1:1787761138.596308:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5680.
00010000:00000010:1.1:1787761139.132294:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac300.
00010000:00000010:1.1:1787761139.132308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7842c00.
00000080:00000010:1.1:1787761139.194394:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caa20.
00000080:00000010:1.1:1787761139.194406:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5680.
00010000:00000010:2.1:1787761139.614371:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac300.
00010000:00000010:2.1:1787761139.614387:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843e00.
00000080:00000010:3.1:1787761139.644301:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae140.
00000080:00000010:3.1:1787761139.644316:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5f80.
00010000:00000010:3.1:1787761140.219179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae600.
00010000:00000010:3.1:1787761140.219195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867ce00.
00000080:00000010:3.1:1787761140.248215:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b40e0.
00000080:00000010:3.1:1787761140.248247:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caa20.
00010000:00000010:2.1:1787761140.800189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac000.
00010000:00000010:2.1:1787761140.800201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7842000.
00000080:00000010:0.1:1787761140.821179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aea20.
00000080:00000010:0.1:1787761140.821194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5bc0.
00010000:00000010:3.1:1787761141.210223:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac100.
00010000:00000010:3.1:1787761141.210239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7941c00.
00000080:00000010:2.1:1787761141.254198:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4cc0.
00000080:00000010:2.1:1787761141.254212:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4340.
00010000:00000010:2.1:1787761141.584248:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac500.
00010000:00000010:2.1:1787761141.584263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843600.
00000080:00000010:0.1:1787761141.642193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae760.
00000080:00000010:0.1:1787761141.642206:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5bc0.
00010000:00000010:3.1:1787761142.069173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87acb00.
00010000:00000010:3.1:1787761142.069188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873f200.
00000080:00000010:3.1:1787761142.102408:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4480.
00000080:00000010:3.1:1787761142.102427:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5bc0.
00010000:00000010:2.1:1787761142.552183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae800.
00010000:00000010:2.1:1787761142.552200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867da00.
00000080:00000010:3.1:1787761142.579849:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae8a0.
00000080:00000010:3.1:1787761142.579866:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cae20.
00010000:00000010:3.1:1787761143.202815:0:257571:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac500.
00010000:00000010:3.1:1787761143.202830:0:257571:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843a00.
00000080:00000010:2.1:1787761143.240184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4480.
00000080:00000010:2.1:1787761143.240197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5860.
00010000:00000010:2.1:1787761143.757470:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac000.
00010000:00000010:2.1:1787761143.757480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843a00.
00000080:00000010:0.1:1787761143.767182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae360.
00000080:00000010:0.1:1787761143.767195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5160.
00010000:00000010:1.1:1787761144.221185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac000.
00010000:00000010:1.1:1787761144.221199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843600.
00000080:00000010:1.1:1787761144.290234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca940.
00000080:00000010:1.1:1787761144.290252:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5160.
00010000:00000010:3.1:1787761144.879308:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac000.
00010000:00000010:3.1:1787761144.879323:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7842000.
00000080:00000010:3.1:1787761144.879329:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b42a0.
00000080:00000010:3.1:1787761144.879338:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5420.
00010000:00000010:2.1:1787761145.524292:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88aec00.
00010000:00000010:2.1:1787761145.524315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867d000.
00000080:00000010:0.1:1787761145.602193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae9e0.
00000080:00000010:0.1:1787761145.602224:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca940.
00010000:00000010:3.1:1787761146.116226:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5a500.
00010000:00000010:3.1:1787761146.116243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843c00.
00000080:00000010:2.1:1787761146.142203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b42a0.
00000080:00000010:2.1:1787761146.142218:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5c80.
00010000:00000010:0.1:1787761146.756592:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5a900.
00010000:00000010:0.1:1787761146.756608:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843e00.
00000080:00000010:0.1:1787761146.797693:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5240.
00000080:00000010:0.1:1787761146.797709:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5e00.
00010000:00000010:2.1:1787761147.293317:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03700.
00010000:00000010:2.1:1787761147.293335:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7941c00.
00000080:00000010:0.1:1787761147.360185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae2c0.
00000080:00000010:0.1:1787761147.360203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4b20.
00010000:00000010:2.1:1787761147.754619:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88aef00.
00010000:00000010:2.1:1787761147.754635:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867c600.
00000080:00000010:0.1:1787761147.816178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae9c0.
00000080:00000010:0.1:1787761147.816191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cae20.
00010000:00000010:2.1:1787761148.164520:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88aed00.
00010000:00000010:2.1:1787761148.164531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867d600.
00000080:00000010:2.1:1787761148.193248:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae040.
00000080:00000010:2.1:1787761148.193266:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca080.
00010000:00000010:3.1:1787761148.853348:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5a400.
00010000:00000010:3.1:1787761148.853364:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843e00.
00000080:00000010:2.1:1787761148.883628:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5e00.
00000080:00000010:3.1:1787761148.884189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4820.
00010000:00000010:3.1:1787761149.426358:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03700.
00010000:00000010:3.1:1787761149.426373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7940c00.
00000080:00000010:2.1:1787761149.508169:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4da0.
00000080:00000010:2.1:1787761149.508182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4560.
00010000:00000010:3.1:1787761150.055181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5a200.
00010000:00000010:3.1:1787761150.055197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7842200.
00000080:00000010:3.1:1787761150.079567:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b43a0.
00000080:00000010:3.1:1787761150.079583:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac55c0.
00010000:00000010:3.1:1787761150.759167:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88aed00.
00010000:00000010:3.1:1787761150.759174:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867c000.
00000080:00000010:3.1:1787761150.769167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b43a0.
00000080:00000010:3.1:1787761150.769179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca780.
00010000:00000010:3.1:1787761151.351297:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5a000.
00010000:00000010:3.1:1787761151.351311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7940a00.
00000080:00000010:3.1:1787761151.373177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4de0.
00000080:00000010:3.1:1787761151.373209:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b43a0.
00010000:00000010:3.1:1787761152.033035:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5aa00.
00010000:00000010:3.1:1787761152.033051:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7842200.
00000080:00000010:3.1:1787761152.089605:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b43a0.
00000080:00000010:3.1:1787761152.089617:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5500.
00010000:00000010:0.1:1787761152.678197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5a300.
00010000:00000010:0.1:1787761152.678210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843e00.
00000080:00000010:0.1:1787761152.722371:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac59c0.
00000080:00000010:0.1:1787761152.722385:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5740.
00010000:00000010:0.1:1787761153.329237:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88aec00.
00010000:00000010:0.1:1787761153.329252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867dc00.
00000080:00000010:0.1:1787761153.397253:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5740.
00000080:00000010:0.1:1787761153.397269:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cabc0.
00000080:00000010:1.1:1787761153.857184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c5640.
00000080:00000010:1.1:1787761153.857205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751020dc2a0.
00000080:00000010:1.1:1787761153.857212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca200.
00000080:00000010:1.1:1787761153.857217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca4a0.
00000080:00000010:1.1:1787761153.857220:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca3a0.
00000080:00000010:1.1:1787761153.857225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d9303e80.
00010000:00000010:1.1:1787761153.857236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843e800.
00010000:00000010:1.1:1787761153.857240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843f000.
00010000:00000010:1.1:1787761153.857245:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6db00.
00010000:00000010:1.1:1787761153.857249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441400.
00010000:00000010:1.1:1787761153.857253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440a00.
00010000:00000010:1.1:1787761153.857256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58e00.
00010000:00000010:1.1:1787761153.857259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f59e00.
00010000:00000010:1.1:1787761153.857263:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90c00.
00010000:00000010:1.1:1787761153.857266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843e000.
00010000:00000010:1.1:1787761153.857269:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518d700.
00010000:00000010:1.1:1787761153.857275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f000.
00010000:00000010:1.1:1787761153.857278:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6df00.
00010000:00000010:1.1:1787761153.857281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440000.
00010000:00000010:1.1:1787761153.857284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ee00.
00010000:00000010:1.1:1787761153.857287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843e400.
00010000:00000010:1.1:1787761153.857289:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d100.
00010000:00000010:1.1:1787761153.857292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539fa00.
00010000:00000010:1.1:1787761153.857294:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6da00.
00010000:00000010:1.1:1787761153.857297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ea00.
00010000:00000010:1.1:1787761153.857300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49800.
00010000:00000010:1.1:1787761153.857304:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d800.
00010000:00000010:1.1:1787761153.857307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441000.
00010000:00000010:1.1:1787761153.857310:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518da00.
00010000:00000010:1.1:1787761153.857312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ec00.
00010000:00000010:3.1:1787761153.862863:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2200.
00010000:00000010:3.1:1787761153.862889:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867cc00.
00000080:00000010:0.1:1787761153.897389:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b43a0.
00000080:00000010:0.1:1787761153.897467:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caec0.
00010000:00000010:0.1:1787761154.425196:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03200.
00010000:00000010:0.1:1787761154.425211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780ca00.
00000080:00000010:0.1:1787761154.464223:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5740.
00000080:00000010:0.1:1787761154.464237:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aee40.
00010000:00000010:3.1:1787761155.051197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2e00.
00010000:00000010:3.1:1787761155.051210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780c000.
00000080:00000010:1.1:1787761155.137152:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4de0.
00000080:00000010:1.1:1787761155.137164:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae8c0.
00010000:00000010:2.1:1787761155.641188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae700.
00010000:00000010:2.1:1787761155.641203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867c200.
00000080:00000010:0.1:1787761155.703180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aeb80.
00000080:00000010:0.1:1787761155.703197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca7a0.
00010000:00000010:0.1:1787761156.156195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae200.
00010000:00000010:0.1:1787761156.156210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867d800.
00000080:00000010:0.1:1787761156.241303:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac59c0.
00000080:00000010:0.1:1787761156.241316:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca720.
00010000:00000010:2.1:1787761156.867164:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae200.
00010000:00000010:2.1:1787761156.867174:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867c200.
00000080:00000010:3.1:1787761156.900193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae3e0.
00000080:00000010:3.1:1787761156.900211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca6e0.
00010000:00000010:3.1:1787761157.428183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae700.
00010000:00000010:3.1:1787761157.428199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867cc00.
00000080:00000010:3.1:1787761157.479208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4dc0.
00000080:00000010:3.1:1787761157.479223:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cae60.
00010000:00000010:0.1:1787761158.098187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5ab00.
00010000:00000010:0.1:1787761158.098204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843600.
00000080:00000010:0.1:1787761158.118159:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac58a0.
00000080:00000010:0.1:1787761158.118174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac52c0.
00010000:00000010:3.1:1787761158.931183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88aec00.
00010000:00000010:3.1:1787761158.931197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867dc00.
00000080:00000010:3.1:1787761158.957118:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4dc0.
00000080:00000010:3.1:1787761158.957158:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cad00.
00010000:00000010:3.1:1787761159.671154:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03d00.
00010000:00000010:3.1:1787761159.671183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7940a00.
00000080:00000010:2.1:1787761159.699978:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4de0.
00000080:00000010:2.1:1787761159.706261:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4dc0.
00010000:00000010:3.1:1787761160.231730:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5ae00.
00010000:00000010:3.1:1787761160.231745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7940a00.
00000080:00000010:0.1:1787761160.319177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4560.
00000080:00000010:0.1:1787761160.319189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b43a0.
00010000:00000010:2.1:1787761160.833382:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88aec00.
00010000:00000010:2.1:1787761160.833393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867d600.
00000080:00000010:0.1:1787761160.917369:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aee40.
00000080:00000010:0.1:1787761160.917388:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caaa0.
00010000:00000010:0.1:1787761160.928350:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d600.
00010000:00000010:0.1:1787761160.928427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441a00.
00010000:00000010:2.1:1787761161.503413:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5af00.
00010000:00000010:2.1:1787761161.503429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867d000.
00000080:00000010:0.1:1787761161.570173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae0e0.
00000080:00000010:0.1:1787761161.570187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca820.
00010000:00000010:3.1:1787761162.107309:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03b00.
00010000:00000010:3.1:1787761162.107324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7940200.
00000080:00000010:3.1:1787761162.141373:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b42a0.
00000080:00000010:3.1:1787761162.141386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4b20.
00010000:00000010:2.1:1787761162.794914:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae200.
00010000:00000010:2.1:1787761162.794929:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797e400.
00000080:00000010:0.1:1787761162.851579:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aee40.
00000080:00000010:0.1:1787761162.851590:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5d80.
00010000:00000010:3.1:1787761163.294375:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5aa00.
00010000:00000010:3.1:1787761163.294389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797ee00.
00000080:00000010:2.1:1787761163.327543:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4b20.
00000080:00000010:2.1:1787761163.327558:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5d80.
00010000:00000010:2.1:1787761163.908515:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae200.
00010000:00000010:2.1:1787761163.908530:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867ce00.
00000080:00000010:0.1:1787761163.977186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae3e0.
00000080:00000010:0.1:1787761163.977199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cade0.
00010000:00000010:3.1:1787761164.580199:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518da00.
00010000:00000010:3.1:1787761164.580215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867ce00.
00000080:00000010:3.1:1787761164.607220:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b42a0.
00000080:00000010:3.1:1787761164.607234:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca7e0.
00010000:00000010:0.1:1787761165.341466:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03600.
00010000:00000010:0.1:1787761165.341481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7941800.
00000080:00000010:0.1:1787761165.364201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5e40.
00000080:00000010:0.1:1787761165.364217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4340.
00010000:00000010:2.1:1787761165.889666:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03200.
00010000:00000010:2.1:1787761165.889682:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7941400.
00000080:00000010:0.1:1787761165.954471:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae620.
00000080:00000010:0.1:1787761165.954488:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4e40.
00010000:00000010:3.1:1787761166.501297:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5a200.
00010000:00000010:3.1:1787761166.501323:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780c600.
00000080:00000010:2.1:1787761166.556177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4340.
00000080:00000010:2.1:1787761166.556191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac58e0.
00010000:00000010:2.1:1787761167.019420:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518da00.
00010000:00000010:2.1:1787761167.019438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867d000.
00000080:00000010:2.1:1787761167.038178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aee40.
00000080:00000010:2.1:1787761167.038191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca7e0.
00010000:00000010:3.1:1787761167.567916:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90c00.
00010000:00000010:3.1:1787761167.567922:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867d600.
00000080:00000010:0.1:1787761167.606182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b43a0.
00000080:00000010:0.1:1787761167.606199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cade0.
00010000:00000010:0.1:1787761168.064193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5a500.
00010000:00000010:0.1:1787761168.064207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797e600.
00000080:00000010:0.1:1787761168.100817:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5540.
00000080:00000010:0.1:1787761168.100840:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5400.
00010000:00000010:3.1:1787761168.661180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5a200.
00010000:00000010:3.1:1787761168.661374:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797e600.
00000080:00000010:1.1:1787761168.732551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4dc0.
00000080:00000010:1.1:1787761168.732568:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5400.
00010000:00000010:3.1:1787761169.160080:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2800.
00010000:00000010:3.1:1787761169.160094:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780ce00.
00000080:00000010:2.1:1787761169.187199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4de0.
00000080:00000010:2.1:1787761169.187217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae9c0.
00010000:00000010:0.1:1787761169.831585:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5a200.
00010000:00000010:0.1:1787761169.831599:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797f600.
00000080:00000010:0.1:1787761169.896170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5c60.
00000080:00000010:0.1:1787761169.896184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5540.
00010000:00000010:0.1:1787761170.568268:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d600.
00010000:00000010:0.1:1787761170.568275:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867cc00.
00000080:00000010:0.1:1787761170.580564:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5540.
00000080:00000010:0.1:1787761170.580575:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca820.
00010000:00000010:3.1:1787761170.939198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5400.
00010000:00000010:3.1:1787761170.939287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797e000.
00000080:00000010:3.1:1787761170.965477:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b43a0.
00000080:00000010:3.1:1787761170.965491:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5d80.
00010000:00000010:2.1:1787761171.334531:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5100.
00010000:00000010:2.1:1787761171.334547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780ce00.
00000080:00000010:0.1:1787761171.384978:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae8c0.
00000080:00000010:0.1:1787761171.384995:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae220.
00010000:00000010:3.1:1787761171.719502:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5000.
00010000:00000010:3.1:1787761171.719516:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797ec00.
00000080:00000010:3.1:1787761171.731234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b43a0.
00000080:00000010:3.1:1787761171.731249:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac52c0.
00010000:00000010:3.1:1787761172.294389:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2800.
00010000:00000010:3.1:1787761172.294398:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780c200.
00000080:00000010:3.1:1787761172.342733:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b43a0.
00000080:00000010:3.1:1787761172.342762:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae8a0.
00010000:00000010:0.1:1787761172.948364:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5a00.
00010000:00000010:0.1:1787761172.948379:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797e200.
00000080:00000010:3.1:1787761172.975976:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac55c0.
00000080:00000010:3.1:1787761172.976060:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5500.
00010000:00000010:3.1:1787761173.626426:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d800.
00010000:00000010:3.1:1787761173.626443:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867cc00.
00000080:00000010:3.1:1787761173.655168:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b43a0.
00000080:00000010:3.1:1787761173.655184:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cae60.
00010000:00000010:2.1:1787761174.337184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d100.
00010000:00000010:2.1:1787761174.337202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867c200.
00000080:00000010:1.1:1787761174.377182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae620.
00000080:00000010:1.1:1787761174.377201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca6e0.
00000080:00000010:3.1:1787761174.958154:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae8a0.
00000080:00000010:3.1:1787761174.958169:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca6e0.
00010000:00000010:2.1:1787761174.958664:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6df00.
00010000:00000010:2.1:1787761174.958672:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867d600.
00010000:00000010:3.1:1787761175.524382:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5300.
00010000:00000010:3.1:1787761175.524398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797e200.
00000080:00000010:3.1:1787761175.567809:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b43a0.
00000080:00000010:3.1:1787761175.567829:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5c80.
00010000:00000010:2.1:1787761175.953239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7b00.
00010000:00000010:2.1:1787761175.953255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7940600.
00000080:00000010:0.1:1787761175.989368:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae180.
00000080:00000010:0.1:1787761175.989379:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4340.
00010000:00000010:2.1:1787761176.381295:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5b00.
00010000:00000010:2.1:1787761176.381309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797e000.
00000080:00000010:0.1:1787761176.415176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aeb80.
00000080:00000010:0.1:1787761176.415190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5160.
00010000:00000010:2.1:1787761176.993177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6df00.
00010000:00000010:2.1:1787761176.993189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867d000.
00000080:00000010:0.1:1787761177.009215:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aee40.
00000080:00000010:0.1:1787761177.009231:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca720.
00010000:00000010:3.1:1787761177.429693:0:257571:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d800.
00010000:00000010:3.1:1787761177.429735:0:257571:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867ce00.
00000080:00000010:2.1:1787761177.458338:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4e40.
00000080:00000010:2.1:1787761177.458362:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caec0.
00010000:00000010:0.1:1787761178.081727:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2800.
00010000:00000010:0.1:1787761178.081743:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780c000.
00000080:00000010:0.1:1787761178.156568:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5bc0.
00000080:00000010:0.1:1787761178.156584:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae9c0.
00010000:00000010:0.1:1787761178.769182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d800.
00010000:00000010:0.1:1787761178.769199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797fc00.
00000080:00000010:0.1:1787761178.779182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5a80.
00000080:00000010:0.1:1787761178.779197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5bc0.
00010000:00000010:0.1:1787761179.391313:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6df00.
00010000:00000010:0.1:1787761179.391326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797fc00.
00000080:00000010:0.1:1787761179.434818:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4de0.
00000080:00000010:0.1:1787761179.434835:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5bc0.
00010000:00000010:2.1:1787761180.053610:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f100.
00010000:00000010:2.1:1787761180.053625:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867c200.
00000080:00000010:2.1:1787761180.075581:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aee40.
00000080:00000010:2.1:1787761180.075594:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca780.
00010000:00000010:0.1:1787761180.805551:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5500.
00010000:00000010:0.1:1787761180.805565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3c400.
00000080:00000010:2.1:1787761180.831203:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5a80.
00000080:00000010:2.1:1787761180.831393:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae560.
00010000:00000010:0.1:1787761181.436289:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4ee00.
00010000:00000010:0.1:1787761181.436302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867c200.
00000080:00000010:0.1:1787761181.484706:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5f80.
00000080:00000010:0.1:1787761181.484726:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caa20.
00010000:00000010:0.1:1787761182.064190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5d00.
00010000:00000010:0.1:1787761182.064214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797f200.
00000080:00000010:0.1:1787761182.083186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac53e0.
00000080:00000010:0.1:1787761182.083215:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5f80.
00010000:00000010:2.1:1787761182.679329:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5d00.
00010000:00000010:2.1:1787761182.679345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7941600.
00000080:00000010:0.1:1787761182.725172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aee60.
00000080:00000010:0.1:1787761182.725183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c50b4de0.
00010000:00000010:3.1:1787761183.210573:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e500.
00010000:00000010:3.1:1787761183.210589:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867cc00.
00000080:00000010:3.1:1787761183.219421:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c62cbe80.
00000080:00000010:3.1:1787761183.219435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca080.
00010000:00000010:2.1:1787761183.871577:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4ed00.
00010000:00000010:2.1:1787761183.871593:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797f200.
00000080:00000010:1.1:1787761183.933876:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae0e0.
00000080:00000010:1.1:1787761183.933896:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5f80.
00010000:00000010:3.1:1787761184.540205:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5b00.
00010000:00000010:3.1:1787761184.540231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797fa00.
00000080:00000010:3.1:1787761184.562188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c62cbe80.
00000080:00000010:3.1:1787761184.562214:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac53e0.
00010000:00000010:2.1:1787761185.237189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7f00.
00010000:00000010:2.1:1787761185.237203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7941400.
00000080:00000010:0.1:1787761185.302225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae620.
00000080:00000010:0.1:1787761185.302242:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3880.
00010000:00000010:0.1:1787761185.951264:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e100.
00010000:00000010:0.1:1787761185.951283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867cc00.
00000080:00000010:0.1:1787761186.010866:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5340.
00000080:00000010:0.1:1787761186.010892:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca780.
00010000:00000010:2.1:1787761186.770625:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f200.
00010000:00000010:2.1:1787761186.770662:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3da00.
00000080:00000010:2.1:1787761186.793884:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3980.
00000080:00000010:2.1:1787761186.793899:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae3e0.
00010000:00000010:1.1:1787761187.457360:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5000.
00010000:00000010:1.1:1787761187.457373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797ee00.
00000080:00000010:1.1:1787761187.457378:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caf00.
00000080:00000010:1.1:1787761187.457389:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5600.
00010000:00000010:2.1:1787761188.101197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5100.
00010000:00000010:2.1:1787761188.101213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3c400.
00000080:00000010:2.1:1787761188.121186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aeb80.
00000080:00000010:2.1:1787761188.121199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae0e0.
00010000:00000010:1.1:1787761188.724233:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f700.
00010000:00000010:1.1:1787761188.724257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3d400.
00000080:00000010:1.1:1787761188.752272:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caf00.
00000080:00000010:1.1:1787761188.752291:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aeb80.
00010000:00000010:1.1:1787761189.324192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5400.
00010000:00000010:1.1:1787761189.324207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3d000.
00000080:00000010:1.1:1787761189.390324:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caf00.
00000080:00000010:1.1:1787761189.390337:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aee60.
00010000:00000010:2.1:1787761189.957753:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f300.
00010000:00000010:2.1:1787761189.957777:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3de00.
00000080:00000010:0.1:1787761189.969443:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aeb80.
00000080:00000010:0.1:1787761189.969457:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84ae620.
00010000:00000010:1.1:1787761190.662522:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5100.
00010000:00000010:1.1:1787761190.662529:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797fa00.
00000080:00000010:0.1:1787761190.696296:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caf00.
00000080:00000010:0.1:1787761190.696309:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac56c0.
00010000:00000010:2.1:1787761191.176770:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e200.
00010000:00000010:2.1:1787761191.176785:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867cc00.
00000080:00000010:3.1:1787761191.194223:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e84aee60.
00000080:00000010:3.1:1787761191.194241:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caec0.
00010000:00000010:0.1:1787761191.725897:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7d00.
00010000:00000010:0.1:1787761191.725912:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7941600.
00000080:00000010:1.1:1787761191.764322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac56c0.
00000080:00000010:1.1:1787761191.764340:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3780.
00010000:00000010:1.1:1787761192.538318:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e000.
00010000:00000010:1.1:1787761192.538331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797e000.
00000080:00000010:1.1:1787761192.572576:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cabe0.
00000080:00000010:1.1:1787761192.572596:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac57a0.
00010000:00000010:2.1:1787761193.364785:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e000.
00010000:00000010:2.1:1787761193.364803:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867cc00.
00000080:00000010:3.1:1787761193.426251:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751f48cea00.
00000080:00000010:3.1:1787761193.426273:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cabe0.
00010000:00000010:0.1:1787761194.135185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870ac00.
00010000:00000010:0.1:1787761194.135201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797e000.
00000080:00000010:0.1:1787761194.155474:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac51e0.
00000080:00000010:0.1:1787761194.155490:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5440.
00010000:00000010:3.1:1787761194.827281:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870ae00.
00010000:00000010:3.1:1787761194.827288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797e000.
00000080:00000010:3.1:1787761194.854411:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3ce0.
00000080:00000010:3.1:1787761194.854421:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5440.
00010000:00000010:3.1:1787761195.215271:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7400.
00010000:00000010:3.1:1787761195.215280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79ae800.
00000080:00000010:3.1:1787761195.240180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3700.
00000080:00000010:3.1:1787761195.240194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3860.
00010000:00000010:0.1:1787761195.642043:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7100.
00010000:00000010:0.1:1787761195.642052:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79ae600.
00000080:00000010:0.1:1787761195.662364:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac51e0.
00000080:00000010:0.1:1787761195.662380:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3700.
00010000:00000010:2.1:1787761196.104188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a500.
00010000:00000010:2.1:1787761196.104216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797fa00.
00000080:00000010:0.1:1787761196.176175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e580.
00000080:00000010:0.1:1787761196.176196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac51e0.
00010000:00000010:2.1:1787761196.786289:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870ad00.
00010000:00000010:2.1:1787761196.786305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3c200.
00000080:00000010:0.1:1787761196.807681:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e740.
00000080:00000010:0.1:1787761196.807697:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198ed80.
00010000:00000010:3.1:1787761197.361177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a900.
00010000:00000010:3.1:1787761197.361192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79ae000.
00000080:00000010:0.1:1787761197.425747:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a31a0.
00000080:00000010:0.1:1787761197.425766:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3920.
00010000:00000010:3.1:1787761198.043211:0:257570:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4ea00.
00010000:00000010:3.1:1787761198.043221:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49800.
00000080:00000010:0.1:1787761198.115505:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a31e0.
00000080:00000010:0.1:1787761198.115523:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cae60.
00010000:00000010:1.1:1787761198.686196:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e300.
00010000:00000010:1.1:1787761198.686211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ec00.
00000080:00000010:1.1:1787761198.708343:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cade0.
00000080:00000010:1.1:1787761198.708361:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caaa0.
00010000:00000010:2.1:1787761199.491197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a100.
00010000:00000010:2.1:1787761199.491213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797fa00.
00000080:00000010:0.1:1787761199.561205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198eca0.
00000080:00000010:0.1:1787761199.561219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5fe0.
00010000:00000010:3.1:1787761200.149192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f800.
00010000:00000010:3.1:1787761200.149207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3d400.
00000080:00000010:3.1:1787761200.167500:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3380.
00000080:00000010:3.1:1787761200.167514:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198eca0.
00010000:00000010:3.1:1787761200.889181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a000.
00010000:00000010:3.1:1787761200.889197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797e000.
00000080:00000010:3.1:1787761200.914622:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3380.
00000080:00000010:3.1:1787761200.914635:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5fe0.
00010000:00000010:2.1:1787761201.378966:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870aa00.
00010000:00000010:2.1:1787761201.378978:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797e000.
00000080:00000010:2.1:1787761201.390184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e3a0.
00000080:00000010:2.1:1787761201.390199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5820.
00010000:00000010:3.1:1787761202.000193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7a00.
00010000:00000010:3.1:1787761202.000207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3c600.
00000080:00000010:0.1:1787761202.066014:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a33a0.
00000080:00000010:0.1:1787761202.066029:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e6e0.
00010000:00000010:3.1:1787761202.634793:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a600.
00010000:00000010:3.1:1787761202.634809:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3d400.
00000080:00000010:2.1:1787761202.668246:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a38e0.
00000080:00000010:2.1:1787761202.668264:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e620.
00010000:00000010:2.1:1787761203.329305:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e800.
00010000:00000010:2.1:1787761203.329321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797e000.
00000080:00000010:2.1:1787761203.417127:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e620.
00000080:00000010:2.1:1787761203.417343:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac56a0.
00010000:00000010:2.1:1787761204.123189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7600.
00010000:00000010:2.1:1787761204.123207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79afe00.
00000080:00000010:0.1:1787761204.205288:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e620.
00000080:00000010:0.1:1787761204.205307:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3dc0.
00010000:00000010:3.1:1787761204.853666:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a800.
00010000:00000010:3.1:1787761204.853677:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3c200.
00000080:00000010:3.1:1787761204.913193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3640.
00000080:00000010:3.1:1787761204.913207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e740.
00010000:00000010:1.1:1787761205.529415:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a800.
00010000:00000010:1.1:1787761205.529428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7941600.
00000080:00000010:1.1:1787761205.595845:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cade0.
00000080:00000010:1.1:1787761205.595861:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac54e0.
00010000:00000010:3.1:1787761206.198205:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4eb00.
00010000:00000010:3.1:1787761206.198214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7941600.
00000080:00000010:3.1:1787761206.206256:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3640.
00000080:00000010:3.1:1787761206.206270:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5360.
00010000:00000010:1.1:1787761206.645191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a000.
00010000:00000010:1.1:1787761206.645205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3c200.
00000080:00000010:1.1:1787761206.704193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cade0.
00000080:00000010:1.1:1787761206.704204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198eda0.
00010000:00000010:2.1:1787761207.090188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a100.
00010000:00000010:2.1:1787761207.090204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3c200.
00000080:00000010:0.1:1787761207.156382:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e560.
00000080:00000010:0.1:1787761207.156398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198efa0.
00010000:00000010:2.1:1787761207.733207:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870ad00.
00010000:00000010:2.1:1787761207.733220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fe200.
00000080:00000010:0.1:1787761207.776698:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198eec0.
00000080:00000010:0.1:1787761207.776714:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5780.
00010000:00000010:0.1:1787761208.452339:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a500.
00010000:00000010:0.1:1787761208.452354:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aea00.
00000080:00000010:0.1:1787761208.524179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5780.
00000080:00000010:0.1:1787761208.524193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3640.
00010000:00000010:2.1:1787761209.156222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7100.
00010000:00000010:2.1:1787761209.156237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79af800.
00000080:00000010:3.1:1787761209.181156:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e740.
00000080:00000010:3.1:1787761209.181168:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3d60.
00010000:00000010:3.1:1787761209.807672:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a500.
00010000:00000010:3.1:1787761209.807690:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b8800.
00000080:00000010:3.1:1787761209.875532:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e6c0.
00000080:00000010:3.1:1787761209.875671:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e620.
00010000:00000010:0.1:1787761210.375721:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a500.
00010000:00000010:0.1:1787761210.375735:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fe400.
00000080:00000010:0.1:1787761210.440332:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5f00.
00000080:00000010:0.1:1787761210.470672:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac52a0.
00010000:00000010:0.1:1787761210.988245:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7b00.
00010000:00000010:0.1:1787761210.988263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aee00.
00000080:00000010:2.1:1787761211.012055:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac52a0.
00000080:00000010:2.1:1787761211.012286:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3040.
00010000:00000010:2.1:1787761211.641449:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4eb00.
00010000:00000010:2.1:1787761211.641459:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ee00.
00000080:00000010:1.1:1787761211.658284:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198eda0.
00000080:00000010:1.1:1787761211.658308:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cae60.
00010000:00000010:2.1:1787761212.183364:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7100.
00010000:00000010:2.1:1787761212.183372:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79af000.
00000080:00000010:0.1:1787761212.205171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e620.
00000080:00000010:0.1:1787761212.205184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3d20.
00010000:00000010:2.1:1787761212.670300:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870ad00.
00010000:00000010:2.1:1787761212.670318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fe400.
00000080:00000010:0.1:1787761212.714632:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e620.
00000080:00000010:0.1:1787761212.714650:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5f00.
00010000:00000010:3.1:1787761213.268524:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4ea00.
00010000:00000010:3.1:1787761213.268535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f000.
00000080:00000010:0.1:1787761213.282189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a34e0.
00000080:00000010:0.1:1787761213.282207:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5cae60.
00010000:00000010:0.1:1787761213.641319:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e200.
00010000:00000010:0.1:1787761213.641329:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ee00.
00000080:00000010:0.1:1787761213.673345:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5ce0.
00000080:00000010:0.1:1787761213.673359:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca720.
00010000:00000010:1.1:1787761214.267183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7100.
00010000:00000010:1.1:1787761214.267199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b8a00.
00000080:00000010:1.1:1787761214.338188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caf00.
00000080:00000010:1.1:1787761214.338205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5ce0.
00010000:00000010:2.1:1787761214.806003:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a800.
00010000:00000010:2.1:1787761214.806018:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ff000.
00000080:00000010:0.1:1787761214.822265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e560.
00000080:00000010:0.1:1787761214.822279:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5aa0.
00010000:00000010:2.1:1787761215.309321:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870ad00.
00010000:00000010:2.1:1787761215.309336:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fe800.
00000080:00000010:0.1:1787761215.328854:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198eda0.
00000080:00000010:0.1:1787761215.328875:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5aa0.
00010000:00000010:2.1:1787761216.009339:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2100.
00010000:00000010:2.1:1787761216.009355:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fe600.
00000080:00000010:0.1:1787761216.067083:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e1e0.
00000080:00000010:0.1:1787761216.067099:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5aa0.
00010000:00000010:2.1:1787761216.572195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2000.
00010000:00000010:2.1:1787761216.572209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b8a00.
00000080:00000010:2.1:1787761216.605160:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e620.
00000080:00000010:2.1:1787761216.605177:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e740.
00010000:00000010:2.1:1787761217.152344:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2700.
00010000:00000010:2.1:1787761217.152361:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79af800.
00000080:00000010:0.1:1787761217.214516:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e740.
00000080:00000010:0.1:1787761217.214534:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e620.
00010000:00000010:0.1:1787761218.119192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f900.
00010000:00000010:0.1:1787761218.119208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b8e00.
00000080:00000010:0.1:1787761218.149180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5aa0.
00000080:00000010:0.1:1787761218.149199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e560.
00010000:00000010:2.1:1787761218.941382:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e900.
00010000:00000010:2.1:1787761218.941398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58e00.
00000080:00000010:0.1:1787761219.007270:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198eca0.
00000080:00000010:0.1:1787761219.007283:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca080.
00010000:00000010:2.1:1787761219.428498:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e500.
00010000:00000010:2.1:1787761219.428507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441a00.
00000080:00000010:0.1:1787761219.443171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198eec0.
00000080:00000010:0.1:1787761219.443183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5ca720.
00010000:00000010:0.1:1787761219.965235:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2600.
00010000:00000010:0.1:1787761219.965247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fe000.
00000080:00000010:0.1:1787761219.995938:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5660.
00000080:00000010:0.1:1787761219.995956:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac54a0.
00010000:00000010:2.1:1787761220.552093:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f900.
00010000:00000010:2.1:1787761220.552108:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aea00.
00000080:00000010:2.1:1787761220.583280:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e1e0.
00000080:00000010:2.1:1787761220.583299:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3080.
00010000:00000010:2.1:1787761221.220359:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f900.
00010000:00000010:2.1:1787761221.220374:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b9e00.
00000080:00000010:2.1:1787761221.261482:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e1e0.
00000080:00000010:2.1:1787761221.261496:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e580.
00010000:00000010:2.1:1787761221.796355:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e500.
00010000:00000010:2.1:1787761221.796370:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440000.
00000080:00000010:0.1:1787761221.873787:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e580.
00000080:00000010:0.1:1787761221.873808:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caec0.
00010000:00000010:2.1:1787761222.445217:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f900.
00010000:00000010:2.1:1787761222.445233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b8000.
00000080:00000010:0.1:1787761222.510355:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e5e0.
00000080:00000010:0.1:1787761222.510366:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5660.
00010000:00000010:0.1:1787761222.911287:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2c00.
00010000:00000010:0.1:1787761222.911300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ffe00.
00000080:00000010:0.1:1787761222.945680:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5520.
00000080:00000010:0.1:1787761222.945752:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5660.
00010000:00000010:0.1:1787761223.504780:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78f7c00.
00010000:00000010:0.1:1787761223.504792:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79ae600.
00000080:00000010:0.1:1787761223.504798:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5660.
00000080:00000010:0.1:1787761223.504806:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3820.
00010000:00000010:0.1:1787761224.175175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e200.
00010000:00000010:0.1:1787761224.175188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441400.
00000080:00000010:0.1:1787761224.203469:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5660.
00000080:00000010:2.1:1787761224.203661:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caec0.
00010000:00000010:2.1:1787761224.924842:0:257570:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2c00.
00010000:00000010:2.1:1787761224.924859:0:257570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b8800.
00000080:00000010:0.1:1787761224.974350:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198efa0.
00000080:00000010:0.1:1787761224.974369:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198eec0.
00010000:00000010:3.1:1787761225.651184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2f00.
00010000:00000010:3.1:1787761225.651199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ffe00.
00000080:00000010:2.1:1787761225.704474:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3200.
00000080:00000010:2.1:1787761225.704493:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5660.
00010000:00000010:1.1:1787761226.496359:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2d00.
00010000:00000010:1.1:1787761226.496379:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fec00.
00000080:00000010:1.1:1787761226.526172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caec0.
00000080:00000010:1.1:1787761226.526187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5520.
00010000:00000010:3.1:1787761227.160205:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2400.
00010000:00000010:3.1:1787761227.160222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fea00.
00000080:00000010:0.1:1787761227.269566:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3440.
00000080:00000010:0.1:1787761227.269595:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5c00.
00010000:00000010:3.1:1787761227.701589:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4eb00.
00010000:00000010:3.1:1787761227.701598:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441400.
00000080:00000010:3.1:1787761227.729158:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a39a0.
00000080:00000010:3.1:1787761227.729171:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5caec0.
00010000:00000010:2.1:1787761228.158402:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4eb00.
00010000:00000010:2.1:1787761228.158417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86bbe00.
00000080:00000010:2.1:1787761228.202033:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198eec0.
00000080:00000010:2.1:1787761228.202049:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3300.
00010000:00000010:2.1:1787761228.641452:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f300.
00010000:00000010:2.1:1787761228.641468:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b8e00.
00000080:00000010:2.1:1787761228.688174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198efa0.
00000080:00000010:2.1:1787761228.688199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e580.
00010000:00000010:3.1:1787761229.126419:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e500.
00010000:00000010:3.1:1787761229.126434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440000.
00000080:00000010:3.1:1787761229.157028:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3f00.
00000080:00000010:3.1:1787761229.157049:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca3a0.
00010000:00000010:3.1:1787761229.542622:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2400.
00010000:00000010:3.1:1787761229.542656:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b8e00.
00000080:00000010:2.1:1787761229.630541:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3ba0.
00000080:00000010:0.1:1787761229.644798:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e560.
00010000:00000010:3.1:1787761230.048366:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e500.
00010000:00000010:3.1:1787761230.048383:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441a00.
00000080:00000010:3.1:1787761230.077194:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3900.
00000080:00000010:3.1:1787761230.077207:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca4a0.
00010000:00000010:3.1:1787761230.503275:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78f7d00.
00010000:00000010:3.1:1787761230.503293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ba200.
00000080:00000010:3.1:1787761230.529359:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3940.
00000080:00000010:3.1:1787761230.529373:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3c60.
00010000:00000010:3.1:1787761231.160172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e500.
00010000:00000010:3.1:1787761231.160187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441a00.
00000080:00000010:3.1:1787761231.182181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3c60.
00000080:00000010:3.1:1787761231.182194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca200.
00010000:00000010:2.1:1787761231.854296:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2f00.
00010000:00000010:2.1:1787761231.854309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ff000.
00000080:00000010:0.1:1787761231.927203:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198e580.
00000080:00000010:0.1:1787761231.927216:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac54c0.
00010000:00000010:2.1:1787761232.567306:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9000.
00010000:00000010:2.1:1787761232.567322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440000.
00000080:00000010:3.1:1787761232.592688:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198efa0.
00000080:00000010:3.1:1787761232.592708:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca200.
00010000:00000010:2.1:1787761233.218180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f700.
00010000:00000010:2.1:1787761233.218203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b9800.
00000080:00000010:0.1:1787761233.255248:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198efa0.
00000080:00000010:0.1:1787761233.255262:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e198ec00.
00010000:00000010:3.1:1787761233.927197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9700.
00010000:00000010:3.1:1787761233.927212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441400.
00000080:00000010:2.1:1787761233.984700:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3940.
00000080:00000010:2.1:1787761233.984717:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca4a0.
00010000:00000010:0.1:1787761234.641330:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2700.
00010000:00000010:0.1:1787761234.641345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ba400.
00000080:00000010:0.1:1787761234.690169:0:1:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5d60.
00000080:00000010:0.1:1787761234.690192:0:1:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3560.
00010000:00000010:3.1:1787761235.333949:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f400.
00010000:00000010:3.1:1787761235.333963:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86bae00.
00000080:00000010:3.1:1787761235.363578:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3840.
00000080:00000010:3.1:1787761235.363594:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3800.
00010000:00000010:1.1:1787761236.121359:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9600.
00010000:00000010:1.1:1787761236.121373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441400.
00000080:00000010:1.1:1787761236.224186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c44c58e0.
00000080:00000010:1.1:1787761236.224201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ea0ca3a0.
00010000:00000010:0.1:1787761236.984441:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2100.
00010000:00000010:0.1:1787761236.984452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fe400.
00000080:00000010:0.1:1787761236.991173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5640.
00000080:00000010:0.1:1787761236.991188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5d60.
00010000:00000010:0.1:1787761237.649489:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2100.
00010000:00000010:0.1:1787761237.649504:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c0e00.
00000080:00000010:0.1:1787761237.704181:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5d60.
00000080:00000010:0.1:1787761237.704202:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c69983c0.
00010000:00000010:3.1:1787761238.243186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9600.
00010000:00000010:3.1:1787761238.243203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fe400.
00000080:00000010:3.1:1787761238.254749:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3800.
00000080:00000010:3.1:1787761238.254769:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5d60.
00010000:00000010:2.1:1787761238.789535:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2700.
00010000:00000010:2.1:1787761238.789543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c1c00.
00000080:00000010:1.1:1787761238.794180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c69987c0.
00000080:00000010:1.1:1787761238.794199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c69983c0.
00010000:00000010:2.1:1787761239.226172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2f00.
00010000:00000010:2.1:1787761239.226186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c1c00.
00000080:00000010:0.1:1787761239.287241:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c69987c0.
00000080:00000010:0.1:1787761239.287261:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c69983c0.
00010000:00000010:3.1:1787761239.755177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9b00.
00010000:00000010:3.1:1787761239.755187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440000.
00000080:00000010:2.1:1787761239.815241:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3840.
00000080:00000010:2.1:1787761239.815258:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0ed5c60.
00010000:00000010:2.1:1787761240.460180:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0fe00.
00010000:00000010:2.1:1787761240.460187:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c1e00.
00000080:00000010:2.1:1787761240.492171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c69983c0.
00000080:00000010:2.1:1787761240.492184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c69987c0.
00010000:00000010:1.1:1787761241.046993:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0fe00.
00010000:00000010:1.1:1787761241.047010:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c1e00.
00000080:00000010:1.1:1787761241.082175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0ed5c60.
00000080:00000010:1.1:1787761241.082189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5920.
00010000:00000010:2.1:1787761241.629292:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6f700.
00010000:00000010:2.1:1787761241.629302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ff000.
00000080:00000010:0.1:1787761241.640495:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6998d60.
00000080:00000010:0.1:1787761241.640511:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5040.
00010000:00000010:3.1:1787761242.339413:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9c00.
00010000:00000010:3.1:1787761242.339420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440000.
00000080:00000010:3.1:1787761242.354399:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3840.
00000080:00000010:3.1:1787761242.354417:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0ed5c60.
00010000:00000010:1.1:1787761242.963181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78f7100.
00010000:00000010:1.1:1787761242.963201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ba800.
00000080:00000010:1.1:1787761243.006161:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d0ed5c60.
00000080:00000010:1.1:1787761243.006174:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3840.
00010000:00000010:2.1:1787761243.527325:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6f300.
00010000:00000010:2.1:1787761243.527341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fec00.
00000080:00000010:0.1:1787761243.596702:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c6998d60.
00000080:00000010:0.1:1787761243.596717:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5040.
00010000:00000010:3.1:1787761244.089748:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9500.
00010000:00000010:3.1:1787761244.089761:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843e400.
00000080:00000010:3.1:1787761244.144476:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3560.
00000080:00000010:3.1:1787761244.144496:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751cfd29bc0.
00010000:00000010:3.1:1787761244.810409:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9d00.
00010000:00000010:3.1:1787761244.810424:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fea00.
00000080:00000010:3.1:1787761244.842610:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3ba0.
00000080:00000010:3.1:1787761244.842631:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5040.
00010000:00000010:0.1:1787761245.550588:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9400.
00010000:00000010:0.1:1787761245.550603:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843f000.
00000080:00000010:1.1:1787761245.593290:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5920.
00000080:00000010:1.1:1787761245.593309:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750f9df7a20.
00010000:00000010:0.1:1787761246.400224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6fb00.
00010000:00000010:0.1:1787761246.400250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86bb600.
00000080:00000010:0.1:1787761246.471163:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5d60.
00000080:00000010:0.1:1787761246.471182:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3300.
00010000:00000010:0.1:1787761247.145201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9100.
00010000:00000010:0.1:1787761247.145214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843e800.
00000080:00000010:0.1:1787761247.184374:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5d60.
00000080:00000010:0.1:1787761247.184396:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750f9df7a20.
00010000:00000010:0.1:1787761247.864303:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9100.
00010000:00000010:0.1:1787761247.864320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843f000.
00000080:00000010:2.1:1787761247.900826:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5d60.
00000080:00000010:2.1:1787761247.900842:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751e0ecc440.
00010000:00000010:3.1:1787761248.601435:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78f7700.
00010000:00000010:3.1:1787761248.601454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86bbc00.
00000080:00000010:0.1:1787761248.657406:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3820.
00000080:00000010:0.1:1787761248.657418:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3200.
00010000:00000010:0.1:1787761249.187001:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6fe00.
00010000:00000010:0.1:1787761249.187015:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fe800.
00000080:00000010:0.1:1787761249.243230:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5b40.
00000080:00000010:0.1:1787761249.243242:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac54c0.
00010000:00000010:2.1:1787761249.990612:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78f7700.
00010000:00000010:2.1:1787761249.990630:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86bbc00.
00000080:00000010:3.1:1787761250.024619:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e91696e0.
00000080:00000010:3.1:1787761250.024636:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3080.
00010000:00000010:3.1:1787761250.567305:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78f7600.
00010000:00000010:3.1:1787761250.567324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ba800.
00000080:00000010:3.1:1787761250.586996:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3d20.
00000080:00000010:3.1:1787761250.587008:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a34e0.
00010000:00000010:1.1:1787761250.958526:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9d00.
00010000:00000010:1.1:1787761250.958534:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843f000.
00000080:00000010:1.1:1787761251.002277:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c2b65f80.
00000080:00000010:1.1:1787761251.002291:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c2b65f00.
00010000:00000010:0.1:1787761251.348189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6fe00.
00010000:00000010:0.1:1787761251.348207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c1600.
00000080:00000010:0.1:1787761251.378176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac54c0.
00000080:00000010:0.1:1787761251.378189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e9169da0.
00010000:00000010:0.1:1787761251.841198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6ff00.
00010000:00000010:0.1:1787761251.841213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fe800.
00000080:00000010:0.1:1787761251.875233:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5c00.
00000080:00000010:0.1:1787761251.875247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac54c0.
00010000:00000010:3.1:1787761252.282100:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59bb800.
00010000:00000010:3.1:1787761252.282112:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c0600.
00000080:00000010:2.1:1787761252.304156:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a34e0.
00000080:00000010:2.1:1787761252.304169:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e91692e0.
00010000:00000010:3.1:1787761252.682368:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59bbc00.
00010000:00000010:3.1:1787761252.682382:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c0400.
00000080:00000010:0.1:1787761252.731795:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3d20.
00000080:00000010:0.1:1787761252.731812:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e9169540.
00010000:00000010:2.1:1787761253.056246:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9d00.
00010000:00000010:2.1:1787761253.056252:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c3000.
00000080:00000010:2.1:1787761253.086162:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e9169360.
00000080:00000010:2.1:1787761253.086175:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8317380.
00010000:00000010:0.1:1787761253.407306:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59bbf00.
00010000:00000010:0.1:1787761253.407321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c0600.
00000080:00000010:0.1:1787761253.423209:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5c00.
00000080:00000010:0.1:1787761253.423223:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e91694c0.
00010000:00000010:2.1:1787761253.847174:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6fd00.
00010000:00000010:2.1:1787761253.847198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ff000.
00000080:00000010:2.1:1787761253.891212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e9169a20.
00000080:00000010:2.1:1787761253.891230:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5520.
00010000:00000010:2.1:1787761254.189750:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6f400.
00010000:00000010:2.1:1787761254.189757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fea00.
00000080:00000010:2.1:1787761254.242195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e9169660.
00000080:00000010:2.1:1787761254.242206:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5660.
00010000:00000010:2.1:1787761254.558181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6f100.
00010000:00000010:2.1:1787761254.558195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ba400.
00000080:00000010:3.1:1787761254.594186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e9169940.
00000080:00000010:3.1:1787761254.594205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750da5a3640.
00010000:00000010:0.1:1787761254.878201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59bbd00.
00010000:00000010:0.1:1787761254.878214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c0400.
00000080:00000010:0.1:1787761254.920598:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5f00.
00000080:00000010:0.1:1787761254.920611:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e9169320.
00010000:00000010:2.1:1787761255.172564:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6f000.
00010000:00000010:2.1:1787761255.172578:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fea00.
00000080:00000010:1.1:1787761255.199182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e91694e0.
00000080:00000010:1.1:1787761255.199196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5f00.
00010000:00000010:1.1:1787761255.426189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6f100.
00010000:00000010:1.1:1787761255.426203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05e00.
00000080:00000010:1.1:1787761255.463249:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8317fc0.
00000080:00000010:1.1:1787761255.463262:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5080.
00010000:00000010:1.1:1787761255.673176:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9e00.
00010000:00000010:1.1:1787761255.673184:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c2600.
00000080:00000010:1.1:1787761255.700186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8317c60.
00000080:00000010:1.1:1787761255.700198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8317fc0.
00010000:00000010:1.1:1787761255.885154:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9c00.
00010000:00000010:1.1:1787761255.885161:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c3200.
00000080:00000010:1.1:1787761255.912173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8317c60.
00000080:00000010:1.1:1787761255.912185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8317fc0.
00010000:00000010:1.1:1787761256.106180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9b00.
00010000:00000010:1.1:1787761256.106192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c3600.
00000080:00000010:0.1:1787761256.138673:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8317c60.
00000080:00000010:0.1:1787761256.138698:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8317fc0.
00010000:00000010:0.1:1787761256.369181:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6fd00.
00010000:00000010:0.1:1787761256.369188:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec04a00.
00000080:00000010:0.1:1787761256.399184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac56a0.
00000080:00000010:0.1:1787761256.399195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac54e0.
00010000:00000010:0.1:1787761256.580182:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6f300.
00010000:00000010:0.1:1787761256.580190:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec04200.
00000080:00000010:0.1:1787761256.603170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac56a0.
00000080:00000010:0.1:1787761256.603180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac54e0.
00010000:00000010:1.1:1787761256.792180:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6f700.
00010000:00000010:1.1:1787761256.792189:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05800.
00000080:00000010:1.1:1787761256.814186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d83175e0.
00000080:00000010:1.1:1787761256.814209:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac54e0.
00010000:00000010:1.1:1787761257.004272:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9600.
00010000:00000010:1.1:1787761257.004285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c2000.
00000080:00000010:1.1:1787761257.022167:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8317180.
00000080:00000010:1.1:1787761257.022180:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d83175e0.
00010000:00000010:1.1:1787761257.257476:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9700.
00010000:00000010:1.1:1787761257.257486:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c3c00.
00000080:00000010:0.1:1787761257.287177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8317860.
00000080:00000010:0.1:1787761257.287189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8317180.
00010000:00000010:0.1:1787761257.507198:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6f100.
00010000:00000010:0.1:1787761257.507207:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05400.
00000080:00000010:0.1:1787761257.534177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5440.
00000080:00000010:0.1:1787761257.534188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5ca0.
00010000:00000010:2.1:1787761257.778181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9000.
00010000:00000010:2.1:1787761257.778196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c3e00.
00000080:00000010:1.1:1787761257.790676:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750e9169b60.
00000080:00000010:1.1:1787761257.790693:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8317860.
00010000:00000010:0.1:1787761258.007335:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78bea00.
00010000:00000010:0.1:1787761258.007343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec04e00.
00000080:00000010:0.1:1787761258.041182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac57a0.
00000080:00000010:0.1:1787761258.041191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5440.
00010000:00000010:1.1:1787761258.248175:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78be600.
00010000:00000010:1.1:1787761258.248181:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05a00.
00000080:00000010:1.1:1787761258.274174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8317860.
00000080:00000010:1.1:1787761258.274185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5440.
00010000:00000010:1.1:1787761258.482043:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9d00.
00010000:00000010:1.1:1787761258.482051:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c2a00.
00000080:00000010:1.1:1787761258.510178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8317c60.
00000080:00000010:1.1:1787761258.510188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8317860.
00010000:00000010:1.1:1787761258.708171:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9100.
00010000:00000010:1.1:1787761258.708177:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c2200.
00000080:00000010:1.1:1787761258.734234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8317c60.
00000080:00000010:1.1:1787761258.734246:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8317860.
00010000:00000010:1.1:1787761258.927173:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6e00.
00010000:00000010:1.1:1787761258.927180:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c3800.
00000080:00000010:1.1:1787761258.953184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8317860.
00000080:00000010:1.1:1787761258.953203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c2b65f80.
00010000:00000010:1.1:1787761259.117178:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6f00.
00010000:00000010:1.1:1787761259.117184:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c2400.
00000080:00000010:1.1:1787761259.142175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c2b65f80.
00000080:00000010:1.1:1787761259.142186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751d8317860.
00010000:00000010:1.1:1787761259.323186:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6500.
00010000:00000010:1.1:1787761259.323194:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c3400.
00000080:00000010:1.1:1787761259.350171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee6fee20.
00000080:00000010:1.1:1787761259.350180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee6fec80.
00010000:00000010:1.1:1787761259.517174:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6d00.
00010000:00000010:1.1:1787761259.517181:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c2e00.
00000080:00000010:1.1:1787761259.545184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee6fee20.
00000080:00000010:1.1:1787761259.545194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee6fec80.
00010000:00000010:1.1:1787761259.714176:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6900.
00010000:00000010:1.1:1787761259.714182:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c3a00.
00000080:00000010:1.1:1787761259.738176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee6fee20.
00000080:00000010:1.1:1787761259.738188:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee6fec80.
00010000:00000010:1.1:1787761259.901167:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6400.
00010000:00000010:1.1:1787761259.901173:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c3e00.
00000080:00000010:1.1:1787761259.924195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee6fee20.
00000080:00000010:1.1:1787761259.924204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee6fec80.
00010000:00000010:1.1:1787761260.092185:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6100.
00010000:00000010:1.1:1787761260.092192:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c3000.
00000080:00000010:1.1:1787761260.119172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee6fee20.
00000080:00000010:1.1:1787761260.119183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee6fec80.
00010000:00000010:1.1:1787761260.277165:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6200.
00010000:00000010:1.1:1787761260.277170:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec06000.
00000080:00000010:1.1:1787761260.301170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee6fee20.
00000080:00000010:1.1:1787761260.301182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee6fec80.
00010000:00000010:1.1:1787761260.455159:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6000.
00010000:00000010:1.1:1787761260.455163:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec07c00.
00000080:00000010:1.1:1787761260.475176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee6fee20.
00000080:00000010:1.1:1787761260.475186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee6fec80.
00010000:00000010:1.1:1787761260.620158:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6700.
00010000:00000010:1.1:1787761260.620162:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec07e00.
00000080:00000010:1.1:1787761260.638183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee6fee20.
00000080:00000010:1.1:1787761260.638193:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee6fec80.
00010000:00000010:1.1:1787761260.782154:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6a00.
00010000:00000010:1.1:1787761260.782158:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec06c00.
00000080:00000010:1.1:1787761260.803159:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee6fee20.
00000080:00000010:1.1:1787761260.807167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee6fec80.
00010000:00000010:1.1:1787761260.955179:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6600.
00010000:00000010:1.1:1787761260.955186:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec06a00.
00000080:00000010:1.1:1787761260.980173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee6fec80.
00000080:00000010:1.1:1787761260.980182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7b41da0.
00010000:00000010:1.1:1787761261.160162:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6300.
00010000:00000010:1.1:1787761261.160168:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec06200.
00000080:00000010:1.1:1787761261.183163:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7b41da0.
00000080:00000010:1.1:1787761261.183175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee6fec80.
00010000:00000010:1.1:1787761261.335153:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6800.
00010000:00000010:1.1:1787761261.335157:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec07800.
00000080:00000010:1.1:1787761261.359174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad960.
00000080:00000010:1.1:1787761261.359182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad4e0.
00010000:00000010:1.1:1787761261.528166:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6b00.
00010000:00000010:1.1:1787761261.528172:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec06400.
00000080:00000010:1.1:1787761261.550158:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad960.
00000080:00000010:1.1:1787761261.550167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad4e0.
00010000:00000010:1.1:1787761261.713164:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6c00.
00010000:00000010:1.1:1787761261.713170:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec07400.
00000080:00000010:1.1:1787761261.740171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad960.
00000080:00000010:1.1:1787761261.740184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad4e0.
00010000:00000010:1.1:1787761261.918154:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49600.
00010000:00000010:1.1:1787761261.918158:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec06e00.
00000080:00000010:1.1:1787761261.940164:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad960.
00000080:00000010:1.1:1787761261.940173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad4e0.
00010000:00000010:1.1:1787761262.112160:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49300.
00010000:00000010:1.1:1787761262.112167:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec07a00.
00000080:00000010:1.1:1787761262.134166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad960.
00000080:00000010:1.1:1787761262.134173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad4e0.
00010000:00000010:1.1:1787761262.312159:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49800.
00010000:00000010:1.1:1787761262.312166:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec07000.
00000080:00000010:1.1:1787761262.340164:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad960.
00000080:00000010:1.1:1787761262.340174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad4e0.
00010000:00000010:1.1:1787761262.495158:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49b00.
00010000:00000010:1.1:1787761262.495163:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec06800.
00000080:00000010:1.1:1787761262.517162:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad960.
00000080:00000010:1.1:1787761262.517172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad4e0.
00010000:00000010:1.1:1787761262.675158:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49c00.
00010000:00000010:1.1:1787761262.675163:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec06600.
00000080:00000010:1.1:1787761262.697160:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad960.
00000080:00000010:1.1:1787761262.697170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad4e0.
00010000:00000010:1.1:1787761262.839153:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49e00.
00010000:00000010:1.1:1787761262.839157:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec07200.
00000080:00000010:1.1:1787761262.857158:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad960.
00000080:00000010:1.1:1787761262.857166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad4e0.
00010000:00000010:1.1:1787761262.999154:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49f00.
00010000:00000010:1.1:1787761262.999158:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec07600.
00000080:00000010:1.1:1787761263.021174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad960.
00000080:00000010:1.1:1787761263.021185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad4e0.
00010000:00000010:1.1:1787761263.180154:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49500.
00010000:00000010:1.1:1787761263.180158:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8795800.
00000080:00000010:1.1:1787761263.202168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad960.
00000080:00000010:1.1:1787761263.202177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad4e0.
00010000:00000010:1.1:1787761263.351157:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49d00.
00010000:00000010:1.1:1787761263.351169:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8794400.
00000080:00000010:1.1:1787761263.373159:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad960.
00000080:00000010:1.1:1787761263.373170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad4e0.
00010000:00000010:1.1:1787761263.524152:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49900.
00010000:00000010:1.1:1787761263.524156:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8795400.
00000080:00000010:1.1:1787761263.545165:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad960.
00000080:00000010:1.1:1787761263.545173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad4e0.
00010000:00000010:1.1:1787761263.682155:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49400.
00010000:00000010:1.1:1787761263.682159:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8794e00.
00000080:00000010:1.1:1787761263.702158:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad960.
00000080:00000010:1.1:1787761263.702164:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad4e0.
00010000:00000010:1.1:1787761263.837159:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49100.
00010000:00000010:1.1:1787761263.837165:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8795a00.
00000080:00000010:1.1:1787761263.858159:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad960.
00000080:00000010:1.1:1787761263.858167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad4e0.
00010000:00000010:1.1:1787761264.020159:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49200.
00010000:00000010:1.1:1787761264.020164:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8795000.
00000080:00000010:1.1:1787761264.043166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad960.
00000080:00000010:1.1:1787761264.043180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad4e0.
00010000:00000010:1.1:1787761264.198167:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49000.
00010000:00000010:1.1:1787761264.198174:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8794800.
00000080:00000010:1.1:1787761264.221167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad960.
00000080:00000010:1.1:1787761264.221178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad4e0.
00010000:00000010:1.1:1787761264.362154:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49700.
00010000:00000010:1.1:1787761264.362158:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8794600.
00000080:00000010:1.1:1787761264.381178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad960.
00000080:00000010:1.1:1787761264.381185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad4e0.
00010000:00000010:1.1:1787761264.531159:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49a00.
00010000:00000010:1.1:1787761264.531164:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8795200.
00000080:00000010:1.1:1787761264.552158:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad960.
00000080:00000010:1.1:1787761264.552167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad4e0.
00010000:00000010:1.1:1787761264.705167:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6100.
00010000:00000010:1.1:1787761264.705177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8795600.
00000080:00000010:0.1:1787761264.728166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad960.
00000080:00000010:0.1:1787761264.728174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad4e0.
00010000:00000010:1.1:1787761264.872156:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78beb00.
00010000:00000010:1.1:1787761264.872161:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05200.
00000080:00000010:1.1:1787761264.889165:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad340.
00000080:00000010:1.1:1787761264.889173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac53e0.
00010000:00000010:1.1:1787761265.019154:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6200.
00010000:00000010:1.1:1787761265.019158:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8795c00.
00000080:00000010:1.1:1787761265.025147:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00000080:00000010:1.1:1787761265.025153:0:304093:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad340.
00010000:00000010:1.1:1787761265.164155:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6000.
00010000:00000010:1.1:1787761265.164159:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8795e00.
00000080:00000010:1.1:1787761265.183158:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761265.183165:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761265.322149:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6700.
00010000:00000010:1.1:1787761265.322153:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8794c00.
00000080:00000010:1.1:1787761265.339160:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761265.339168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761265.486152:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6a00.
00010000:00000010:1.1:1787761265.486156:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8794a00.
00000080:00000010:1.1:1787761265.506158:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761265.506165:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761265.659162:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6600.
00010000:00000010:1.1:1787761265.659167:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8794200.
00000080:00000010:1.1:1787761265.685168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761265.685179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761265.860154:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6300.
00010000:00000010:1.1:1787761265.860158:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8794000.
00000080:00000010:1.1:1787761265.881158:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761265.881165:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761266.020153:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6800.
00010000:00000010:1.1:1787761266.020157:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8797a00.
00000080:00000010:1.1:1787761266.041167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761266.041176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761266.197158:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6b00.
00010000:00000010:1.1:1787761266.197164:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8797000.
00000080:00000010:1.1:1787761266.217159:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761266.217166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761266.379159:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6c00.
00010000:00000010:1.1:1787761266.379174:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8796800.
00000080:00000010:1.1:1787761266.407166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761266.407174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761266.558153:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6e00.
00010000:00000010:1.1:1787761266.558156:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8796600.
00000080:00000010:1.1:1787761266.579169:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761266.579179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761266.729156:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6f00.
00010000:00000010:1.1:1787761266.729161:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8797200.
00000080:00000010:1.1:1787761266.748166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761266.748175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761266.906153:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6500.
00010000:00000010:1.1:1787761266.906157:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8797600.
00000080:00000010:1.1:1787761266.926160:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761266.926167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761267.084159:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6d00.
00010000:00000010:1.1:1787761267.084165:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8796000.
00000080:00000010:1.1:1787761267.105161:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761267.105168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761267.257279:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6900.
00010000:00000010:1.1:1787761267.257286:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8797c00.
00000080:00000010:1.1:1787761267.282159:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761267.282168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761267.442155:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6400.
00010000:00000010:1.1:1787761267.442159:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8797e00.
00000080:00000010:1.1:1787761267.462169:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761267.462176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761267.619157:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780200.
00010000:00000010:1.1:1787761267.619163:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8796c00.
00000080:00000010:1.1:1787761267.647162:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761267.647170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761267.790159:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780000.
00010000:00000010:1.1:1787761267.790164:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8796a00.
00000080:00000010:1.1:1787761267.810159:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761267.810166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761267.952154:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780700.
00010000:00000010:1.1:1787761267.952157:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8796200.
00000080:00000010:1.1:1787761267.970159:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761267.970167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761268.115159:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780a00.
00010000:00000010:1.1:1787761268.115163:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8797800.
00000080:00000010:1.1:1787761268.135169:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761268.135177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761268.288152:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780600.
00010000:00000010:1.1:1787761268.288155:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8796400.
00000080:00000010:1.1:1787761268.309165:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761268.309175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761268.455154:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780300.
00010000:00000010:1.1:1787761268.455158:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8797400.
00000080:00000010:1.1:1787761268.475162:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761268.475171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761268.617159:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780800.
00010000:00000010:1.1:1787761268.617164:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8796e00.
00000080:00000010:1.1:1787761268.635166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761268.635174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761268.815157:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780b00.
00010000:00000010:1.1:1787761268.815162:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5398c00.
00000080:00000010:1.1:1787761268.844161:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761268.844169:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761269.036156:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780c00.
00010000:00000010:1.1:1787761269.036161:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5398a00.
00000080:00000010:1.1:1787761269.064165:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761269.064174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761269.216155:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780e00.
00010000:00000010:1.1:1787761269.216159:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5398200.
00000080:00000010:1.1:1787761269.239158:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761269.239166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761269.391155:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780f00.
00010000:00000010:1.1:1787761269.391160:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5399800.
00000080:00000010:1.1:1787761269.411177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761269.411185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761269.578156:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780500.
00010000:00000010:1.1:1787761269.578161:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5398400.
00000080:00000010:1.1:1787761269.596166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761269.596175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761269.755157:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780d00.
00010000:00000010:1.1:1787761269.755163:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5399400.
00000080:00000010:1.1:1787761269.777159:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761269.777167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761269.948159:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780900.
00010000:00000010:1.1:1787761269.948165:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5398e00.
00000080:00000010:1.1:1787761269.977166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761269.977177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761270.138158:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780400.
00010000:00000010:1.1:1787761270.138164:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5399a00.
00000080:00000010:1.1:1787761270.159159:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761270.159167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761270.315160:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780100.
00010000:00000010:1.1:1787761270.315166:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5399000.
00000080:00000010:1.1:1787761270.344167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761270.344176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761270.500158:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2e800.
00010000:00000010:1.1:1787761270.500164:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5398800.
00000080:00000010:1.1:1787761270.526170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761270.526181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761270.685155:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2eb00.
00010000:00000010:1.1:1787761270.685159:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5398600.
00000080:00000010:1.1:1787761270.707170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761270.707180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761270.861158:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2ec00.
00010000:00000010:1.1:1787761270.861163:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5399200.
00000080:00000010:1.1:1787761270.880161:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761270.880168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761271.042155:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2ee00.
00010000:00000010:1.1:1787761271.042160:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5399600.
00000080:00000010:1.1:1787761271.062170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761271.062180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761271.208153:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2ef00.
00010000:00000010:1.1:1787761271.208157:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5398000.
00000080:00000010:1.1:1787761271.228164:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761271.228170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761271.382155:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2e500.
00010000:00000010:1.1:1787761271.382161:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5399c00.
00000080:00000010:1.1:1787761271.411164:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761271.411174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761271.557161:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2ed00.
00010000:00000010:1.1:1787761271.557167:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5399e00.
00000080:00000010:1.1:1787761271.581164:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761271.581173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761271.749157:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2e900.
00010000:00000010:1.1:1787761271.749163:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539b800.
00000080:00000010:1.1:1787761271.777173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761271.777182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761271.929158:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2e400.
00010000:00000010:1.1:1787761271.929162:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539a400.
00000080:00000010:1.1:1787761271.948171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761271.948180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761272.097158:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2e200.
00010000:00000010:1.1:1787761272.097164:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539b400.
00000080:00000010:1.1:1787761272.115169:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761272.115178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761272.266158:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2e000.
00010000:00000010:1.1:1787761272.266164:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ae00.
00000080:00000010:1.1:1787761272.294164:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761272.294172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761272.441152:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2e700.
00010000:00000010:1.1:1787761272.441156:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ba00.
00000080:00000010:1.1:1787761272.459162:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761272.459169:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761272.597156:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2ea00.
00010000:00000010:1.1:1787761272.597161:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539b000.
00000080:00000010:1.1:1787761272.613160:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761272.613167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761272.767161:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2e600.
00010000:00000010:1.1:1787761272.767167:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539a800.
00000080:00000010:1.1:1787761272.794167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761272.794175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761272.960162:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2e300.
00010000:00000010:1.1:1787761272.960167:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539a600.
00000080:00000010:1.1:1787761272.982163:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761272.982173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761273.138156:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2e100.
00010000:00000010:1.1:1787761273.138161:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539b200.
00000080:00000010:1.1:1787761273.167166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761273.167178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761273.336156:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8587000.
00010000:00000010:1.1:1787761273.336161:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539b600.
00000080:00000010:1.1:1787761273.364162:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761273.364170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761273.501157:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8587700.
00010000:00000010:1.1:1787761273.501162:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539a000.
00000080:00000010:1.1:1787761273.524163:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761273.524171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761273.680161:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8587a00.
00010000:00000010:1.1:1787761273.680168:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539bc00.
00000080:00000010:1.1:1787761273.700168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761273.700181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761273.874168:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8587600.
00010000:00000010:1.1:1787761273.874176:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539be00.
00000080:00000010:1.1:1787761273.894159:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761273.894166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761274.051158:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8587300.
00010000:00000010:1.1:1787761274.051164:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ac00.
00000080:00000010:1.1:1787761274.077165:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761274.077173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761274.235167:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8587800.
00010000:00000010:1.1:1787761274.235173:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539aa00.
00000080:00000010:1.1:1787761274.262175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761274.262184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:1.1:1787761274.426153:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8587b00.
00010000:00000010:1.1:1787761274.426157:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539a200.
00000080:00000010:1.1:1787761274.446159:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ad000.
00000080:00000010:1.1:1787761274.446165:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:0.1:1787761274.604165:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8587c00.
00010000:00000010:0.1:1787761274.604170:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f861d400.
00000080:00000010:0.1:1787761274.627168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5240.
00000080:00000010:0.1:1787761274.627175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9750ee7ade00.
00010000:00000010:0.1:1787761274.799158:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78bee00.
00010000:00000010:0.1:1787761274.799163:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05600.
00000080:00000010:0.1:1787761274.829166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac55c0.
00000080:00000010:0.1:1787761274.829175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5240.
00010000:00000010:0.1:1787761275.002161:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78bef00.
00010000:00000010:0.1:1787761275.002166:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05200.
00000080:00000010:0.1:1787761275.031178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac55c0.
00000080:00000010:0.1:1787761275.031186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5240.
00010000:00000010:0.1:1787761275.198157:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78be500.
00010000:00000010:0.1:1787761275.198163:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec04800.
00000080:00000010:0.1:1787761275.228164:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac55c0.
00000080:00000010:0.1:1787761275.228175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5240.
00010000:00000010:0.1:1787761275.368153:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78bed00.
00010000:00000010:0.1:1787761275.368157:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05000.
00000080:00000010:0.1:1787761275.388166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac55c0.
00000080:00000010:0.1:1787761275.388174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5240.
00010000:00000010:0.1:1787761275.537157:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78be900.
00010000:00000010:0.1:1787761275.537163:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05a00.
00000080:00000010:0.1:1787761275.557160:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac55c0.
00000080:00000010:0.1:1787761275.557166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5240.
00010000:00000010:0.1:1787761275.720153:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78be400.
00010000:00000010:0.1:1787761275.720157:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05400.
00000080:00000010:0.1:1787761275.738165:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac55c0.
00000080:00000010:0.1:1787761275.738172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5240.
00010000:00000010:0.1:1787761275.875161:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78be100.
00010000:00000010:0.1:1787761275.875166:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05e00.
00000080:00000010:0.1:1787761275.894167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac55c0.
00000080:00000010:0.1:1787761275.894178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5240.
00010000:00000010:0.1:1787761276.043162:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78be200.
00010000:00000010:0.1:1787761276.043168:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05800.
00000080:00000010:0.1:1787761276.066169:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac55c0.
00000080:00000010:0.1:1787761276.066178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5240.
00010000:00000010:0.1:1787761276.204153:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78be000.
00010000:00000010:0.1:1787761276.204157:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05c00.
00000080:00000010:0.1:1787761276.222172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac55c0.
00000080:00000010:0.1:1787761276.222180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5240.
00010000:00000010:0.1:1787761276.363153:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78be700.
00010000:00000010:0.1:1787761276.363157:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f861fe00.
00000080:00000010:0.1:1787761276.384157:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac55c0.
00000080:00000010:0.1:1787761276.384163:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5240.
00010000:00000010:0.1:1787761276.518157:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78beb00.
00010000:00000010:0.1:1787761276.518161:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f861ec00.
00000080:00000010:0.1:1787761276.533160:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac55c0.
00000080:00000010:0.1:1787761276.533166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5240.
00010000:00000010:0.1:1787761276.676154:0:304093:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78be300.
00010000:00000010:0.1:1787761276.676158:0:304093:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f861ea00.
00000080:00000010:0.1:1787761276.693160:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac55c0.
00000080:00000010:0.1:1787761276.693166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9751c7ac5240.
00010000:00000010:0.1:1787761276.834157:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78be600.
00010000:00000010:0.1:1787761276.834163:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f861e200.
00000800:00000200:3.1:1787761337.148226:0:257567:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 3, ksnr_connected 6 ksnr_max_conns 1
00000800:00000200:3.1:1787761337.148230:0:257567:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 2, ksnr_connected 2 ksnr_max_conns 1
00000800:00000200:3.1:1787761337.148232:0:257567:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 0 ksnr_max_conns 1
00000800:00000200:2.1:1787761337.405102:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761341.501105:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761341.501443:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761341.501779:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761341.501979:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761346.621515:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761346.621928:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761346.622187:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761346.622357:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761351.741052:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761351.741067:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 4, 2 secs later)
00000800:00000200:2.1:1787761351.741827:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761351.741831:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 4, 2 secs later)
00000800:00000200:2.1:1787761351.742069:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761351.742071:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 4, 2 secs later)
00000800:00000200:2.1:1787761351.742356:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761351.742358:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 4, 2 secs later)
00000800:00000200:2.1:1787761356.860971:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761356.861351:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761356.861514:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761356.861636:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761361.980979:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761361.980983:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 8, 6 secs later)
00000800:00000200:2.1:1787761361.982202:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761361.982204:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 8, 6 secs later)
00000800:00000200:2.1:1787761361.982408:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761361.982410:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 8, 6 secs later)
00000800:00000200:2.1:1787761361.982684:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761361.982686:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 8, 6 secs later)
00010000:00000010:1.1:1787761366.727178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1900.
00010000:00000010:1.1:1787761366.727205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441200.
00010000:00000010:1.1:1787761366.727208:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0500.
00010000:00000010:1.1:1787761366.727211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f57800.
00010000:00000010:1.1:1787761366.727214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1100.
00010000:00000010:1.1:1787761366.727216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888400.
00010000:00000010:1.1:1787761366.727218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1800.
00010000:00000010:1.1:1787761366.727221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b800.
00010000:00000010:1.1:1787761366.727224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080900.
00010000:00000010:1.1:1787761366.727226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923800.
00010000:00000010:1.1:1787761366.727229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080500.
00010000:00000010:1.1:1787761366.727231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56600.
00010000:00000010:1.1:1787761366.727233:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0700.
00010000:00000010:1.1:1787761366.727235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56400.
00010000:00000010:1.1:1787761366.727237:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0200.
00010000:00000010:1.1:1787761366.727240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56c00.
00010000:00000010:1.1:1787761366.727242:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0e00.
00010000:00000010:1.1:1787761366.727244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56a00.
00010000:00000010:1.1:1787761366.727246:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1000.
00010000:00000010:1.1:1787761366.727248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798a400.
00010000:00000010:1.1:1787761366.727250:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0f00.
00010000:00000010:1.1:1787761366.727252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f57c00.
00010000:00000010:1.1:1787761366.727254:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0800.
00010000:00000010:1.1:1787761366.727255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798a000.
00010000:00000010:1.1:1787761366.727257:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1d00.
00010000:00000010:1.1:1787761366.727258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b400.
00010000:00000010:1.1:1787761366.727260:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0c00.
00010000:00000010:1.1:1787761366.727262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f57600.
00010000:00000010:1.1:1787761366.727264:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0000.
00010000:00000010:1.1:1787761366.727265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56800.
00010000:00000010:1.1:1787761366.727267:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1f00.
00010000:00000010:1.1:1787761366.727268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798ac00.
00010000:00000010:1.1:1787761366.727270:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1300.
00010000:00000010:1.1:1787761366.727272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798be00.
00010000:00000010:1.1:1787761366.727274:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419500.
00010000:00000010:1.1:1787761366.727276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f57400.
00010000:00000010:1.1:1787761366.731176:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0900.
00010000:00000010:1.1:1787761366.731183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f57e00.
00010000:00000010:1.1:1787761366.731187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1200.
00010000:00000010:1.1:1787761366.731189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7889000.
00010000:00000010:1.1:1787761366.731191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0600.
00010000:00000010:1.1:1787761366.731193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56000.
00010000:00000010:1.1:1787761366.731195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9d00.
00010000:00000010:1.1:1787761366.731198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f57000.
00010000:00000010:1.1:1787761366.731200:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419100.
00010000:00000010:1.1:1787761366.731201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7889600.
00010000:00000010:1.1:1787761366.731203:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419000.
00010000:00000010:1.1:1787761366.731206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f57a00.
00010000:00000010:1.1:1787761366.731208:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975202160100.
00010000:00000010:1.1:1787761366.731211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56e00.
00010000:00000010:1.1:1787761366.731213:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975202160900.
00010000:00000010:1.1:1787761366.731214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f56200.
00010000:00000010:1.1:1787761366.731216:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1c00.
00010000:00000010:1.1:1787761366.731218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b200.
00010000:00000010:1.1:1787761366.731220:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975202160400.
00010000:00000010:1.1:1787761366.731222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06200.
00010000:00000010:1.1:1787761366.731224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975202160d00.
00010000:00000010:1.1:1787761366.731226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad07c00.
00010000:00000010:1.1:1787761366.731228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6b00.
00010000:00000010:1.1:1787761366.731230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06800.
00010000:00000010:1.1:1787761366.731232:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6e00.
00010000:00000010:1.1:1787761366.731234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06e00.
00010000:00000010:1.1:1787761366.731236:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6600.
00010000:00000010:1.1:1787761366.731238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06c00.
00010000:00000010:1.1:1787761366.731240:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9700.
00010000:00000010:1.1:1787761366.731241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798ae00.
00010000:00000010:1.1:1787761366.731243:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6f00.
00010000:00000010:1.1:1787761366.731245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad07400.
00010000:00000010:1.1:1787761366.731246:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6800.
00010000:00000010:1.1:1787761366.731248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad07800.
00010000:00000010:1.1:1787761366.731250:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6700.
00010000:00000010:1.1:1787761366.731251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06a00.
00010000:00000010:1.1:1787761366.731253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9e00.
00010000:00000010:1.1:1787761366.731255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798ba00.
00010000:00000010:1.1:1787761366.731256:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9300.
00010000:00000010:1.1:1787761366.731258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b000.
00010000:00000010:1.1:1787761366.731259:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6c00.
00010000:00000010:1.1:1787761366.731261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06400.
00010000:00000010:1.1:1787761366.731263:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6500.
00010000:00000010:1.1:1787761366.731265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06600.
00010000:00000010:1.1:1787761366.731267:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6d00.
00010000:00000010:1.1:1787761366.731269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad07600.
00010000:00000010:1.1:1787761366.731271:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6400.
00010000:00000010:1.1:1787761366.731273:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad07e00.
00010000:00000010:1.1:1787761366.731275:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9500.
00010000:00000010:1.1:1787761366.731276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798b600.
00010000:00000010:1.1:1787761366.731278:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9f00.
00010000:00000010:1.1:1787761366.731279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798a200.
00010000:00000010:1.1:1787761366.731281:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9000.
00010000:00000010:1.1:1787761366.731283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798a800.
00010000:00000010:1.1:1787761366.731284:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080e00.
00010000:00000010:1.1:1787761366.731286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad07000.
00010000:00000010:1.1:1787761366.731288:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080300.
00010000:00000010:1.1:1787761366.731289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2400.
00010000:00000010:1.1:1787761366.731291:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9b00.
00010000:00000010:1.1:1787761366.731293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798aa00.
00010000:00000010:1.1:1787761366.731295:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419a00.
00010000:00000010:1.1:1787761366.731296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3c00.
00010000:00000010:1.1:1787761366.731298:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419700.
00010000:00000010:1.1:1787761366.731300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7889800.
00010000:00000010:1.1:1787761366.731301:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419200.
00010000:00000010:1.1:1787761366.731303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888000.
00010000:00000010:1.1:1787761366.731304:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9800.
00010000:00000010:1.1:1787761366.731306:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798a600.
00010000:00000010:1.1:1787761366.731307:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6a00.
00010000:00000010:1.1:1787761366.731309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3a00.
00010000:00000010:1.1:1787761366.731311:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9100.
00010000:00000010:1.1:1787761366.731312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d798bc00.
00010000:00000010:1.1:1787761366.731314:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419400.
00010000:00000010:1.1:1787761366.731316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c800.
00010000:00000010:1.1:1787761366.731318:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419900.
00010000:00000010:1.1:1787761366.731320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c200.
00010000:00000010:1.1:1787761366.731322:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6300.
00010000:00000010:1.1:1787761366.731324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3000.
00010000:00000010:1.1:1787761366.731326:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6200.
00010000:00000010:1.1:1787761366.731328:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3800.
00010000:00000010:1.1:1787761366.731330:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6900.
00010000:00000010:1.1:1787761366.731331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104d400.
00010000:00000010:1.1:1787761366.731333:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6000.
00010000:00000010:1.1:1787761366.731335:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3400.
00010000:00000010:1.1:1787761366.731336:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9c00.
00010000:00000010:1.1:1787761366.731338:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e600.
00010000:00000010:1.1:1787761366.731340:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84a6100.
00010000:00000010:1.1:1787761366.731342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2a00.
00010000:00000010:1.1:1787761366.731343:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9900.
00010000:00000010:1.1:1787761366.731345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f200.
00010000:00000010:1.1:1787761366.731347:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19b00.
00010000:00000010:1.1:1787761366.731349:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3600.
00010000:00000010:1.1:1787761366.731351:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9400.
00010000:00000010:1.1:1787761366.731352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ee00.
00010000:00000010:1.1:1787761366.731354:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9a00.
00010000:00000010:1.1:1787761366.731355:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e400.
00010000:00000010:1.1:1787761366.731357:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419d00.
00010000:00000010:1.1:1787761366.731359:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2600.
00010000:00000010:1.1:1787761366.731360:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19c00.
00010000:00000010:1.1:1787761366.731362:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2200.
00010000:00000010:1.1:1787761366.731364:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19d00.
00010000:00000010:1.1:1787761366.731365:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104ca00.
00010000:00000010:1.1:1787761366.731367:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9200.
00010000:00000010:1.1:1787761366.731369:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f000.
00010000:00000010:1.1:1787761366.731371:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79a9600.
00010000:00000010:1.1:1787761366.731373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4fe00.
00010000:00000010:1.1:1787761366.731375:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19100.
00010000:00000010:1.1:1787761366.731377:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2c00.
00010000:00000010:1.1:1787761366.731379:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19600.
00010000:00000010:1.1:1787761366.731381:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3e00.
00010000:00000010:1.1:1787761366.731383:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419f00.
00010000:00000010:1.1:1787761366.731385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104d000.
00010000:00000010:1.1:1787761366.731387:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419e00.
00010000:00000010:1.1:1787761366.731388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f600.
00010000:00000010:1.1:1787761366.731406:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19300.
00010000:00000010:1.1:1787761366.731410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a3200.
00010000:00000010:1.1:1787761366.731412:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5400.
00010000:00000010:1.1:1787761366.731414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f400.
00010000:00000010:1.1:1787761366.731416:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19a00.
00010000:00000010:1.1:1787761366.731418:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2000.
00010000:00000010:1.1:1787761366.731420:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5c00.
00010000:00000010:1.1:1787761366.731421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ea00.
00010000:00000010:1.1:1787761366.731423:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5e00.
00010000:00000010:1.1:1787761366.731425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4fa00.
00010000:00000010:1.1:1787761366.731426:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19f00.
00010000:00000010:1.1:1787761366.731428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2800.
00010000:00000010:1.1:1787761366.731430:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5800.
00010000:00000010:1.1:1787761366.731431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4ec00.
00010000:00000010:1.1:1787761366.731433:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419c00.
00010000:00000010:1.1:1787761366.731434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c400.
00010000:00000010:1.1:1787761366.731436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19200.
00010000:00000010:1.1:1787761366.731437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f12a2e00.
00010000:00000010:1.1:1787761366.731439:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19700.
00010000:00000010:1.1:1787761366.731441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846ca00.
00010000:00000010:1.1:1787761366.731443:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5100.
00010000:00000010:1.1:1787761366.731444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104c600.
00010000:00000010:1.1:1787761366.731446:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19800.
00010000:00000010:1.1:1787761366.731447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c600.
00010000:00000010:1.1:1787761366.731449:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19500.
00010000:00000010:1.1:1787761366.731451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d000.
00010000:00000010:1.1:1787761366.731453:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fee19e00.
00010000:00000010:1.1:1787761366.731455:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d600.
00010000:00000010:1.1:1787761366.731457:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f400.
00010000:00000010:1.1:1787761366.731459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d400.
00010000:00000010:1.1:1787761366.731461:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f500.
00010000:00000010:1.1:1787761366.731462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846d800.
00010000:00000010:1.1:1787761366.731464:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1ff00.
00010000:00000010:1.1:1787761366.731466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846cc00.
00010000:00000010:1.1:1787761366.731467:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1fa00.
00010000:00000010:1.1:1787761366.731469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c800.
00010000:00000010:1.1:1787761366.731470:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5300.
00010000:00000010:1.1:1787761366.731472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c200.
00010000:00000010:1.1:1787761366.731473:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5d00.
00010000:00000010:1.1:1787761366.731475:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4fc00.
00010000:00000010:1.1:1787761366.731476:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8ac5600.
00010000:00000010:1.1:1787761366.731478:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4e200.
00010000:00000010:1.1:1787761366.731479:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d8a1f100.
00010000:00000010:1.1:1787761366.731481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f846c400.
00000800:00000200:3.1:1787761367.101587:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787761367.101591:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 8, 1 secs later)
00000800:00000200:3.1:1787761367.102050:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787761367.102053:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 8, 1 secs later)
00000800:00000200:3.1:1787761367.102301:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787761367.102303:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 8, 1 secs later)
00000800:00000200:3.1:1787761367.102529:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787761367.102531:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 8, 1 secs later)
00000800:00000200:3.1:1787761372.220976:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787761372.221287:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787761372.221481:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787761372.221632:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00010000:00000010:2.1:1787761376.966178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49d00.
00010000:00000010:2.1:1787761376.966186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923400.
00010000:00000010:2.1:1787761376.966188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4b400.
00010000:00000010:2.1:1787761376.966191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48800.
00010000:00000010:2.1:1787761376.966193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4bd00.
00010000:00000010:2.1:1787761376.966195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48a00.
00010000:00000010:2.1:1787761376.966197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4b500.
00010000:00000010:2.1:1787761376.966199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49000.
00010000:00000010:2.1:1787761376.966201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49100.
00010000:00000010:2.1:1787761376.966203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe469000.
00010000:00000010:2.1:1787761376.966206:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742b00.
00010000:00000010:2.1:1787761376.966208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48400.
00010000:00000010:2.1:1787761376.966211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5e00.
00010000:00000010:2.1:1787761376.966213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6c000.
00010000:00000010:2.1:1787761376.966215:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742300.
00010000:00000010:2.1:1787761376.966217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6dc00.
00010000:00000010:2.1:1787761376.966218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742400.
00010000:00000010:2.1:1787761376.966220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f3e00.
00010000:00000010:2.1:1787761376.966222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5900.
00010000:00000010:2.1:1787761376.966224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d400.
00010000:00000010:2.1:1787761376.966226:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742700.
00010000:00000010:2.1:1787761376.966228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f2000.
00010000:00000010:2.1:1787761376.966229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080c00.
00010000:00000010:2.1:1787761376.966231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe469c00.
00010000:00000010:2.1:1787761376.966233:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5d00.
00010000:00000010:2.1:1787761376.966235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6ce00.
00010000:00000010:2.1:1787761376.966237:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5200.
00010000:00000010:2.1:1787761376.966238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468e00.
00010000:00000010:2.1:1787761376.966240:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49900.
00010000:00000010:2.1:1787761376.966242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f2a00.
00010000:00000010:2.1:1787761376.966244:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5600.
00010000:00000010:2.1:1787761376.966246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d000.
00010000:00000010:2.1:1787761376.966247:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742c00.
00010000:00000010:2.1:1787761376.966249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f3c00.
00010000:00000010:2.1:1787761376.966251:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742200.
00010000:00000010:2.1:1787761376.966252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6de00.
00010000:00000010:2.1:1787761376.966254:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742000.
00010000:00000010:2.1:1787761376.966255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f2200.
00010000:00000010:2.1:1787761376.966258:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742800.
00010000:00000010:2.1:1787761376.966259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f2c00.
00010000:00000010:2.1:1787761376.966261:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5700.
00010000:00000010:2.1:1787761376.966263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6c200.
00010000:00000010:2.1:1787761376.966265:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5a00.
00010000:00000010:2.1:1787761376.966267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed9200.
00010000:00000010:2.1:1787761376.966268:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5100.
00010000:00000010:2.1:1787761376.966270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8200.
00010000:00000010:2.1:1787761376.966272:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742600.
00010000:00000010:2.1:1787761376.966274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f3800.
00010000:00000010:2.1:1787761376.966276:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9b00.
00010000:00000010:2.1:1787761376.966277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8400.
00010000:00000010:2.1:1787761376.966279:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742a00.
00010000:00000010:2.1:1787761376.966281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f3600.
00010000:00000010:2.1:1787761376.966282:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080d00.
00010000:00000010:2.1:1787761376.966284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888800.
00010000:00000010:2.1:1787761376.966286:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742900.
00010000:00000010:2.1:1787761376.966287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752021f2400.
00010000:00000010:2.1:1787761376.966289:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9c00.
00010000:00000010:2.1:1787761376.966291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7889200.
00010000:00000010:2.1:1787761376.966293:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9d00.
00010000:00000010:2.1:1787761376.966294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8e00.
00010000:00000010:2.1:1787761376.966296:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742500.
00010000:00000010:2.1:1787761376.966298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888a00.
00010000:00000010:2.1:1787761376.970178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742100.
00010000:00000010:2.1:1787761376.970184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c0031600.
00010000:00000010:2.1:1787761376.970187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9e00.
00010000:00000010:2.1:1787761376.970190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8000.
00010000:00000010:2.1:1787761376.970191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751da742e00.
00010000:00000010:2.1:1787761376.970193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c0031e00.
00010000:00000010:2.1:1787761376.970196:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080100.
00010000:00000010:2.1:1787761376.970198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c0031200.
00010000:00000010:2.1:1787761376.970201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90000.
00010000:00000010:2.1:1787761376.970203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c0031c00.
00010000:00000010:2.1:1787761376.970205:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9900.
00010000:00000010:2.1:1787761376.970207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed9800.
00010000:00000010:2.1:1787761376.970209:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49c00.
00010000:00000010:2.1:1787761376.970211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8a00.
00010000:00000010:2.1:1787761376.970212:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90700.
00010000:00000010:2.1:1787761376.970214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c0030000.
00010000:00000010:2.1:1787761376.970216:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90500.
00010000:00000010:2.1:1787761376.970218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c0030600.
00010000:00000010:2.1:1787761376.970219:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9800.
00010000:00000010:2.1:1787761376.970221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8c00.
00010000:00000010:2.1:1787761376.970223:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90200.
00010000:00000010:2.1:1787761376.970224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58000.
00010000:00000010:2.1:1787761376.970226:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9f00.
00010000:00000010:2.1:1787761376.970228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed9a00.
00010000:00000010:2.1:1787761376.970229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9500.
00010000:00000010:2.1:1787761376.970231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7889a00.
00010000:00000010:2.1:1787761376.970233:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9400.
00010000:00000010:2.1:1787761376.970234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed9000.
00010000:00000010:2.1:1787761376.970236:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90900.
00010000:00000010:2.1:1787761376.970237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58c00.
00010000:00000010:2.1:1787761376.970239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9600.
00010000:00000010:2.1:1787761376.970240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed9600.
00010000:00000010:2.1:1787761376.970242:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080a00.
00010000:00000010:2.1:1787761376.970244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922000.
00010000:00000010:2.1:1787761376.970245:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1cb9300.
00010000:00000010:2.1:1787761376.970247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8600.
00010000:00000010:2.1:1787761376.970249:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101e00.
00010000:00000010:2.1:1787761376.970251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed8800.
00010000:00000010:2.1:1787761376.970253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101a00.
00010000:00000010:2.1:1787761376.970255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed9e00.
00010000:00000010:2.1:1787761376.970257:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080600.
00010000:00000010:2.1:1787761376.970259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922a00.
00010000:00000010:2.1:1787761376.970260:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080700.
00010000:00000010:2.1:1787761376.970262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58600.
00010000:00000010:2.1:1787761376.970264:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080200.
00010000:00000010:2.1:1787761376.970266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923c00.
00010000:00000010:2.1:1787761376.970267:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101600.
00010000:00000010:2.1:1787761376.970269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888e00.
00010000:00000010:2.1:1787761376.970271:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101400.
00010000:00000010:2.1:1787761376.970272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e0ed9c00.
00010000:00000010:2.1:1787761376.970274:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101100.
00010000:00000010:2.1:1787761376.970276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440400.
00010000:00000010:2.1:1787761376.970277:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101200.
00010000:00000010:2.1:1787761376.970279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440600.
00010000:00000010:2.1:1787761376.970281:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101c00.
00010000:00000010:2.1:1787761376.970282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7889c00.
00010000:00000010:2.1:1787761376.970284:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101f00.
00010000:00000010:2.1:1787761376.970286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441c00.
00010000:00000010:2.1:1787761376.970287:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101b00.
00010000:00000010:2.1:1787761376.970289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441600.
00010000:00000010:2.1:1787761376.970290:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49200.
00010000:00000010:2.1:1787761376.970292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922200.
00010000:00000010:2.1:1787761376.970293:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101d00.
00010000:00000010:2.1:1787761376.970295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440800.
00010000:00000010:2.1:1787761376.970296:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90100.
00010000:00000010:2.1:1787761376.970298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f59000.
00010000:00000010:2.1:1787761376.970300:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101900.
00010000:00000010:2.1:1787761376.970302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440e00.
00010000:00000010:2.1:1787761376.970303:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90600.
00010000:00000010:2.1:1787761376.970305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f59400.
00010000:00000010:2.1:1787761376.970306:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90300.
00010000:00000010:2.1:1787761376.970308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58200.
00010000:00000010:2.1:1787761376.970310:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0a00.
00010000:00000010:2.1:1787761376.970312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f59200.
00010000:00000010:2.1:1787761376.970314:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0100.
00010000:00000010:2.1:1787761376.970316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f59a00.
00010000:00000010:2.1:1787761376.970332:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0b00.
00010000:00000010:2.1:1787761376.970334:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f59600.
00010000:00000010:2.1:1787761376.970336:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49600.
00010000:00000010:2.1:1787761376.970338:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922600.
00010000:00000010:2.1:1787761376.970339:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101300.
00010000:00000010:2.1:1787761376.970341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440c00.
00010000:00000010:2.1:1787761376.970343:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0d00.
00010000:00000010:2.1:1787761376.970344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f59c00.
00010000:00000010:2.1:1787761376.970346:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101500.
00010000:00000010:2.1:1787761376.970347:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888600.
00010000:00000010:2.1:1787761376.970349:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49f00.
00010000:00000010:2.1:1787761376.970350:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7889400.
00010000:00000010:2.1:1787761376.970352:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0300.
00010000:00000010:2.1:1787761376.970353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440200.
00010000:00000010:2.1:1787761376.970355:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419b00.
00010000:00000010:2.1:1787761376.970357:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923e00.
00010000:00000010:2.1:1787761376.970358:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080400.
00010000:00000010:2.1:1787761376.970360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923600.
00010000:00000010:2.1:1787761376.970362:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419800.
00010000:00000010:2.1:1787761376.970363:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f59800.
00010000:00000010:2.1:1787761376.970365:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8419600.
00010000:00000010:2.1:1787761376.970366:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58800.
00010000:00000010:2.1:1787761376.970368:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0101700.
00010000:00000010:2.1:1787761376.970369:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441800.
00010000:00000010:2.1:1787761376.970371:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85e0400.
00010000:00000010:2.1:1787761376.970372:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f57200.
00010000:00000010:2.1:1787761376.970374:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1a00.
00010000:00000010:2.1:1787761376.970376:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7889e00.
00010000:00000010:2.1:1787761376.970378:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1700.
00010000:00000010:2.1:1787761376.970380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888200.
00010000:00000010:2.1:1787761376.970382:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1500.
00010000:00000010:2.1:1787761376.970384:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7888c00.
00010000:00000010:2.1:1787761376.970386:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0ed1e00.
00010000:00000010:2.1:1787761376.970387:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441e00.
00000800:00000200:0.1:1787761377.340953:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761377.340961:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 16, 14 secs later)
00000800:00000200:0.1:1787761377.341601:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761377.341605:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 16, 14 secs later)
00000800:00000200:0.1:1787761377.341872:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761377.341874:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 16, 14 secs later)
00000800:00000200:0.1:1787761377.342094:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761377.342097:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 16, 14 secs later)
00000800:00000200:0.1:1787761382.460776:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761382.460785:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 16, 9 secs later)
00000800:00000200:0.1:1787761382.461472:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761382.461475:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 16, 9 secs later)
00000800:00000200:0.1:1787761382.461701:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761382.461703:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 16, 9 secs later)
00000800:00000200:0.1:1787761382.461987:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761382.461989:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 16, 9 secs later)
00010000:00000010:2.1:1787761387.207171:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5d00.
00010000:00000010:2.1:1787761387.207178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da514800.
00010000:00000010:2.1:1787761387.207182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4e17600.
00010000:00000010:2.1:1787761387.207185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6aec00.
00010000:00000010:2.1:1787761387.207188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4e17200.
00010000:00000010:2.1:1787761387.207191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6afe00.
00010000:00000010:2.1:1787761387.207194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5700.
00010000:00000010:2.1:1787761387.207197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e800.
00010000:00000010:2.1:1787761387.207200:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4e17c00.
00010000:00000010:2.1:1787761387.207203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e400.
00010000:00000010:2.1:1787761387.207206:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5200.
00010000:00000010:2.1:1787761387.207209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783ae00.
00010000:00000010:2.1:1787761387.207212:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183000.
00010000:00000010:2.1:1787761387.207215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f200.
00010000:00000010:2.1:1787761387.207218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3908d00.
00010000:00000010:2.1:1787761387.207221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6ae800.
00010000:00000010:2.1:1787761387.207224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5f00.
00010000:00000010:2.1:1787761387.207227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e200.
00010000:00000010:2.1:1787761387.207229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5a00.
00010000:00000010:2.1:1787761387.207232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539e600.
00010000:00000010:2.1:1787761387.207235:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5c00.
00010000:00000010:2.1:1787761387.207237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f400.
00010000:00000010:2.1:1787761387.207240:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5400.
00010000:00000010:2.1:1787761387.207243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814200.
00010000:00000010:2.1:1787761387.207246:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5600.
00010000:00000010:2.1:1787761387.207263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f600.
00010000:00000010:2.1:1787761387.207267:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5500.
00010000:00000010:2.1:1787761387.207270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814400.
00010000:00000010:2.1:1787761387.207273:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183500.
00010000:00000010:2.1:1787761387.207276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe469200.
00010000:00000010:2.1:1787761387.207280:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183100.
00010000:00000010:2.1:1787761387.207283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe469800.
00010000:00000010:2.1:1787761387.207287:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3908500.
00010000:00000010:2.1:1787761387.207290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6af800.
00010000:00000010:2.1:1787761387.207293:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3908800.
00010000:00000010:2.1:1787761387.207296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468000.
00010000:00000010:2.1:1787761387.207299:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3908f00.
00010000:00000010:2.1:1787761387.207301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6ae200.
00010000:00000010:2.1:1787761387.207304:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3908a00.
00010000:00000010:2.1:1787761387.207307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6af200.
00010000:00000010:2.1:1787761387.207310:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5300.
00010000:00000010:2.1:1787761387.207313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb4f800.
00010000:00000010:2.1:1787761387.207316:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5800.
00010000:00000010:2.1:1787761387.207318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539fc00.
00010000:00000010:2.1:1787761387.207321:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3908900.
00010000:00000010:2.1:1787761387.207324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6aea00.
00010000:00000010:2.1:1787761387.207326:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3908300.
00010000:00000010:2.1:1787761387.207329:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6af000.
00010000:00000010:2.1:1787761387.207332:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3908c00.
00010000:00000010:2.1:1787761387.207334:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6afc00.
00010000:00000010:2.1:1787761387.207337:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3908e00.
00010000:00000010:2.1:1787761387.207340:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520104ce00.
00010000:00000010:2.1:1787761387.211169:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4bf00.
00010000:00000010:2.1:1787761387.211176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6af400.
00010000:00000010:2.1:1787761387.211179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4bc00.
00010000:00000010:2.1:1787761387.211183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468800.
00010000:00000010:2.1:1787761387.211186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4bb00.
00010000:00000010:2.1:1787761387.211190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6ae400.
00010000:00000010:2.1:1787761387.211193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4b200.
00010000:00000010:2.1:1787761387.211196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6ae000.
00010000:00000010:2.1:1787761387.211198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183e00.
00010000:00000010:2.1:1787761387.211201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468200.
00010000:00000010:2.1:1787761387.211204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d500.
00010000:00000010:2.1:1787761387.211208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dad06000.
00010000:00000010:2.1:1787761387.211211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183600.
00010000:00000010:2.1:1787761387.211213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468400.
00010000:00000010:2.1:1787761387.211216:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183a00.
00010000:00000010:2.1:1787761387.211219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922c00.
00010000:00000010:2.1:1787761387.211222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d700.
00010000:00000010:2.1:1787761387.211225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d200.
00010000:00000010:2.1:1787761387.211228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d200.
00010000:00000010:2.1:1787761387.211230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d600.
00010000:00000010:2.1:1787761387.211233:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6dc00.
00010000:00000010:2.1:1787761387.211236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6ca00.
00010000:00000010:2.1:1787761387.211239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4b100.
00010000:00000010:2.1:1787761387.211242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6afa00.
00010000:00000010:2.1:1787761387.211245:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4b700.
00010000:00000010:2.1:1787761387.211260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49200.
00010000:00000010:2.1:1787761387.211264:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d900.
00010000:00000010:2.1:1787761387.211267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48600.
00010000:00000010:2.1:1787761387.211270:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080f00.
00010000:00000010:2.1:1787761387.211273:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922400.
00010000:00000010:2.1:1787761387.211276:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6de00.
00010000:00000010:2.1:1787761387.211279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6c800.
00010000:00000010:2.1:1787761387.211282:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5b00.
00010000:00000010:2.1:1787761387.211286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468c00.
00010000:00000010:2.1:1787761387.211289:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5300.
00010000:00000010:2.1:1787761387.211292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe469a00.
00010000:00000010:2.1:1787761387.211296:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080000.
00010000:00000010:2.1:1787761387.211299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7922e00.
00010000:00000010:2.1:1787761387.211302:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4b900.
00010000:00000010:2.1:1787761387.211305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49a00.
00010000:00000010:2.1:1787761387.211308:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4ba00.
00010000:00000010:2.1:1787761387.211311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48c00.
00010000:00000010:2.1:1787761387.211313:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183d00.
00010000:00000010:2.1:1787761387.211316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468600.
00010000:00000010:2.1:1787761387.211319:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4b800.
00010000:00000010:2.1:1787761387.211321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49e00.
00010000:00000010:2.1:1787761387.211324:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5000.
00010000:00000010:2.1:1787761387.211327:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6c400.
00010000:00000010:2.1:1787761387.211329:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5500.
00010000:00000010:2.1:1787761387.211332:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6d800.
00010000:00000010:2.1:1787761387.211334:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183900.
00010000:00000010:2.1:1787761387.211337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe469600.
00010000:00000010:2.1:1787761387.211340:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49500.
00010000:00000010:2.1:1787761387.211343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe469e00.
00010000:00000010:2.1:1787761387.211345:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c5080800.
00010000:00000010:2.1:1787761387.211348:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe468a00.
00010000:00000010:2.1:1787761387.211350:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5c00.
00010000:00000010:2.1:1787761387.211353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6da00.
00010000:00000010:2.1:1787761387.211355:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5800.
00010000:00000010:2.1:1787761387.211358:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923200.
00010000:00000010:2.1:1787761387.211360:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4b300.
00010000:00000010:2.1:1787761387.211363:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49c00.
00010000:00000010:2.1:1787761387.211365:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4be00.
00010000:00000010:2.1:1787761387.211368:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49600.
00010000:00000010:2.1:1787761387.211371:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac49e00.
00010000:00000010:2.1:1787761387.211373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe469400.
00010000:00000010:2.1:1787761387.211376:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5f00.
00010000:00000010:2.1:1787761387.211379:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6cc00.
00010000:00000010:2.1:1787761387.211381:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a5400.
00010000:00000010:2.1:1787761387.211384:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa6c600.
00010000:00000010:2.1:1787761387.211387:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c2b4b600.
00010000:00000010:2.1:1787761387.211390:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d48000.
00000800:00000200:0.1:1787761387.580793:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761387.580803:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 16, 4 secs later)
00000800:00000200:0.1:1787761387.581465:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761387.581467:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 16, 4 secs later)
00000800:00000200:0.1:1787761387.581662:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761387.581664:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 16, 4 secs later)
00000800:00000200:0.1:1787761387.581922:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761387.581924:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 16, 4 secs later)
00000800:00000200:0.1:1787761392.701069:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761392.701606:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761392.701762:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761392.701928:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00010000:00000010:2.1:1787761396.422224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183200.
00010000:00000010:2.1:1787761396.422254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783b800.
00010000:00000010:2.1:1787761396.422260:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1400.
00010000:00000010:2.1:1787761396.422264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a6400.
00010000:00000010:2.1:1787761396.422268:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5200.
00010000:00000010:2.1:1787761396.422271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec815a00.
00010000:00000010:2.1:1787761396.422274:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183f00.
00010000:00000010:2.1:1787761396.422277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783be00.
00010000:00000010:2.1:1787761396.422280:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1100.
00010000:00000010:2.1:1787761396.422282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a6e00.
00010000:00000010:2.1:1787761396.422285:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d100.
00010000:00000010:2.1:1787761396.422296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581c00.
00010000:00000010:2.1:1787761396.422299:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d500.
00010000:00000010:2.1:1787761396.422301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a7000.
00010000:00000010:2.1:1787761396.422303:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3dd00.
00010000:00000010:2.1:1787761396.422305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783b000.
00010000:00000010:2.1:1787761396.422307:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3da00.
00010000:00000010:2.1:1787761396.422309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783a800.
00010000:00000010:2.1:1787761396.422311:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5900.
00010000:00000010:2.1:1787761396.422313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec815800.
00010000:00000010:2.1:1787761396.422315:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5300.
00010000:00000010:2.1:1787761396.422317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814a00.
00010000:00000010:2.1:1787761396.422318:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3db00.
00010000:00000010:2.1:1787761396.422320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783b200.
00010000:00000010:2.1:1787761396.422322:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183300.
00010000:00000010:2.1:1787761396.422324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec815000.
00010000:00000010:2.1:1787761396.422326:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1700.
00010000:00000010:2.1:1787761396.422328:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a7600.
00010000:00000010:2.1:1787761396.422330:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d800.
00010000:00000010:2.1:1787761396.422332:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da514a00.
00010000:00000010:2.1:1787761396.422334:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5c00.
00010000:00000010:2.1:1787761396.422336:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec815200.
00010000:00000010:2.1:1787761396.422338:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d700.
00010000:00000010:2.1:1787761396.422340:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783b600.
00010000:00000010:2.1:1787761396.422342:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5e00.
00010000:00000010:2.1:1787761396.422344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814000.
00010000:00000010:2.1:1787761396.422345:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d300.
00010000:00000010:2.1:1787761396.422347:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da515c00.
00010000:00000010:2.1:1787761396.422349:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1b00.
00010000:00000010:2.1:1787761396.422359:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a7200.
00010000:00000010:2.1:1787761396.422361:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d400.
00010000:00000010:2.1:1787761396.422363:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da515e00.
00010000:00000010:2.1:1787761396.422364:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d600.
00010000:00000010:2.1:1787761396.422366:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783ac00.
00010000:00000010:2.1:1787761396.422368:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d200.
00010000:00000010:2.1:1787761396.422369:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da514400.
00010000:00000010:2.1:1787761396.422371:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1d00.
00010000:00000010:2.1:1787761396.422373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a6800.
00010000:00000010:2.1:1787761396.422375:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1300.
00010000:00000010:2.1:1787761396.422377:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a7a00.
00010000:00000010:2.1:1787761396.422378:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d000.
00010000:00000010:2.1:1787761396.422380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da515200.
00010000:00000010:2.1:1787761396.422382:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3de00.
00010000:00000010:2.1:1787761396.422384:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783b400.
00010000:00000010:2.1:1787761396.422385:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1e00.
00010000:00000010:2.1:1787761396.422387:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a7400.
00010000:00000010:2.1:1787761396.422388:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1c00.
00010000:00000010:2.1:1787761396.422390:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a7800.
00010000:00000010:2.1:1787761396.422392:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4e17000.
00010000:00000010:2.1:1787761396.422394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a6a00.
00010000:00000010:2.1:1787761396.422395:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4e17500.
00010000:00000010:2.1:1787761396.422397:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a6c00.
00010000:00000010:2.1:1787761396.422415:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3df00.
00010000:00000010:2.1:1787761396.422418:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da515000.
00010000:00000010:2.1:1787761396.422427:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4e17d00.
00010000:00000010:2.1:1787761396.422441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da515800.
00010000:00000010:2.1:1787761396.422444:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183c00.
00010000:00000010:2.1:1787761396.422447:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec815e00.
00010000:00000010:2.1:1787761396.422450:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3dc00.
00010000:00000010:2.1:1787761396.422452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da515a00.
00010000:00000010:2.1:1787761396.422456:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4e17300.
00010000:00000010:2.1:1787761396.422458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a7c00.
00010000:00000010:2.1:1787761396.422461:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4e17100.
00010000:00000010:2.1:1787761396.422464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6ae600.
00010000:00000010:2.1:1787761396.422467:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4e17900.
00010000:00000010:2.1:1787761396.422469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783bc00.
00010000:00000010:2.1:1787761396.422472:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa3d900.
00010000:00000010:2.1:1787761396.422475:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783a000.
00010000:00000010:2.1:1787761396.422480:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5b00.
00010000:00000010:2.1:1787761396.422483:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da514200.
00010000:00000010:2.1:1787761396.422486:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5400.
00010000:00000010:2.1:1787761396.422489:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da515400.
00010000:00000010:2.1:1787761396.422492:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183800.
00010000:00000010:2.1:1787761396.422494:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783ba00.
00010000:00000010:2.1:1787761396.422497:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5000.
00010000:00000010:2.1:1787761396.422500:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec815400.
00010000:00000010:2.1:1787761396.422503:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5100.
00010000:00000010:2.1:1787761396.422505:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da514e00.
00010000:00000010:2.1:1787761396.422508:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca183700.
00010000:00000010:2.1:1787761396.422510:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783aa00.
00010000:00000010:2.1:1787761396.422511:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5e00.
00010000:00000010:2.1:1787761396.422514:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da514600.
00010000:00000010:2.1:1787761396.422517:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4e17700.
00010000:00000010:2.1:1787761396.422519:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750df6aee00.
00010000:00000010:2.1:1787761396.422523:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c44a5900.
00010000:00000010:2.1:1787761396.422532:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751da514c00.
00000800:00000200:0.1:1787761397.820934:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761397.820938:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 30 secs later)
00000800:00000200:0.1:1787761397.821795:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761397.821799:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 30 secs later)
00000800:00000200:0.1:1787761397.822204:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761397.822206:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 30 secs later)
00000800:00000200:0.1:1787761397.822481:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761397.822483:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 30 secs later)
00000800:00000200:1.1:1787761402.941190:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761402.941199:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 25 secs later)
00000800:00000200:1.1:1787761402.941914:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761402.941917:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 25 secs later)
00000800:00000200:1.1:1787761402.942177:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761402.942179:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 25 secs later)
00000800:00000200:1.1:1787761402.942385:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761402.942386:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 25 secs later)
00010000:00000010:2.1:1787761406.661171:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47800.
00010000:00000010:2.1:1787761406.661179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e8400.
00010000:00000010:2.1:1787761406.661206:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4500.
00010000:00000010:2.1:1787761406.661209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8770600.
00010000:00000010:2.1:1787761406.661211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841be00.
00010000:00000010:2.1:1787761406.661214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581600.
00010000:00000010:2.1:1787761406.661216:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4400.
00010000:00000010:2.1:1787761406.661218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581e00.
00010000:00000010:2.1:1787761406.661220:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4200.
00010000:00000010:2.1:1787761406.661222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa38600.
00010000:00000010:2.1:1787761406.661224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47600.
00010000:00000010:2.1:1787761406.661226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e8200.
00010000:00000010:2.1:1787761406.661228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47a00.
00010000:00000010:2.1:1787761406.661230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e8c00.
00010000:00000010:2.1:1787761406.661232:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47300.
00010000:00000010:2.1:1787761406.661233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e9c00.
00010000:00000010:2.1:1787761406.661236:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5700.
00010000:00000010:2.1:1787761406.661238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa39c00.
00010000:00000010:2.1:1787761406.661239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841bc00.
00010000:00000010:2.1:1787761406.661242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580400.
00010000:00000010:2.1:1787761406.661244:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5100.
00010000:00000010:2.1:1787761406.661246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814c00.
00010000:00000010:2.1:1787761406.661247:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b800.
00010000:00000010:2.1:1787761406.661249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580a00.
00010000:00000010:2.1:1787761406.661251:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b600.
00010000:00000010:2.1:1787761406.661253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843ec00.
00010000:00000010:2.1:1787761406.661255:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841ba00.
00010000:00000010:2.1:1787761406.661258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843fa00.
00010000:00000010:2.1:1787761406.661260:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b700.
00010000:00000010:2.1:1787761406.661261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580600.
00010000:00000010:2.1:1787761406.661264:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5b00.
00010000:00000010:2.1:1787761406.661266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec815c00.
00010000:00000010:2.1:1787761406.661268:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4f00.
00010000:00000010:2.1:1787761406.661270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814800.
00010000:00000010:2.1:1787761406.661272:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47500.
00010000:00000010:2.1:1787761406.661274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843f800.
00010000:00000010:2.1:1787761406.661276:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b200.
00010000:00000010:2.1:1787761406.661278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581200.
00010000:00000010:2.1:1787761406.661280:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738c00.
00010000:00000010:2.1:1787761406.661283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843ea00.
00010000:00000010:2.1:1787761406.661284:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4c00.
00010000:00000010:2.1:1787761406.661286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580000.
00010000:00000010:2.1:1787761406.661288:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738500.
00010000:00000010:2.1:1787761406.661290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843e200.
00010000:00000010:2.1:1787761406.661292:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738300.
00010000:00000010:2.1:1787761406.661294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843e600.
00010000:00000010:2.1:1787761406.661296:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1600.
00010000:00000010:2.1:1787761406.661298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e8600.
00010000:00000010:2.1:1787761406.661300:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5500.
00010000:00000010:2.1:1787761406.661302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814e00.
00010000:00000010:2.1:1787761406.661304:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738100.
00010000:00000010:2.1:1787761406.661306:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580200.
00010000:00000010:2.1:1787761406.661307:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738200.
00010000:00000010:2.1:1787761406.661309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580e00.
00010000:00000010:2.1:1787761406.661311:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738a00.
00010000:00000010:2.1:1787761406.661313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581000.
00010000:00000010:2.1:1787761406.661314:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738800.
00010000:00000010:2.1:1787761406.661316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580800.
00010000:00000010:2.1:1787761406.661318:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5a00.
00010000:00000010:2.1:1787761406.661319:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec815600.
00010000:00000010:2.1:1787761406.661321:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738400.
00010000:00000010:2.1:1787761406.661323:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843f400.
00010000:00000010:2.1:1787761406.661325:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1f00.
00010000:00000010:2.1:1787761406.661327:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e8800.
00010000:00000010:2.1:1787761406.661329:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738900.
00010000:00000010:2.1:1787761406.661331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843fc00.
00010000:00000010:2.1:1787761406.661333:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738d00.
00010000:00000010:2.1:1787761406.661335:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581a00.
00010000:00000010:2.1:1787761406.661337:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1200.
00010000:00000010:2.1:1787761406.661338:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e9000.
00010000:00000010:2.1:1787761406.661340:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738f00.
00010000:00000010:2.1:1787761406.661342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843ee00.
00010000:00000010:2.1:1787761406.661343:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1a00.
00010000:00000010:2.1:1787761406.661345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a7e00.
00010000:00000010:2.1:1787761406.661347:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1800.
00010000:00000010:2.1:1787761406.661349:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843f200.
00010000:00000010:2.1:1787761406.661350:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1000.
00010000:00000010:2.1:1787761406.661352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a6600.
00010000:00000010:2.1:1787761406.661354:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1500.
00010000:00000010:2.1:1787761406.661355:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a6200.
00010000:00000010:2.1:1787761406.661357:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5800.
00010000:00000010:2.1:1787761406.661359:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843fe00.
00010000:00000010:2.1:1787761406.661360:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4ca1900.
00010000:00000010:2.1:1787761406.661368:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9750ec814600.
00010000:00000010:2.1:1787761406.661374:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738e00.
00010000:00000010:2.1:1787761406.661376:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581400.
00010000:00000010:2.1:1787761406.661378:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738b00.
00010000:00000010:2.1:1787761406.661380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783a400.
00010000:00000010:2.1:1787761406.661381:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4000.
00010000:00000010:2.1:1787761406.661383:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8581800.
00010000:00000010:2.1:1787761406.661385:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738600.
00010000:00000010:2.1:1787761406.661386:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79a6000.
00010000:00000010:2.1:1787761406.661391:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738700.
00010000:00000010:2.1:1787761406.661393:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783a200.
00010000:00000010:2.1:1787761406.661396:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8738000.
00010000:00000010:2.1:1787761406.661398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8580c00.
00000800:00000200:1.1:1787761408.060691:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761408.060695:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 20 secs later)
00000800:00000200:1.1:1787761408.061296:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761408.061297:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 20 secs later)
00000800:00000200:1.1:1787761408.061479:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761408.061481:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 20 secs later)
00000800:00000200:1.1:1787761408.061650:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761408.061652:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 20 secs later)
00000800:00000200:1.1:1787761413.180702:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761413.180709:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 15 secs later)
00000800:00000200:1.1:1787761413.181289:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761413.181290:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 15 secs later)
00000800:00000200:1.1:1787761413.181476:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761413.181478:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 15 secs later)
00000800:00000200:1.1:1787761413.181650:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761413.181651:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 15 secs later)
00010000:00000010:2.1:1787761416.900179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c500.
00010000:00000010:2.1:1787761416.900187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874aa00.
00010000:00000010:2.1:1787761416.900191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb100.
00010000:00000010:2.1:1787761416.900194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068aa00.
00010000:00000010:2.1:1787761416.900196:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47f00.
00010000:00000010:2.1:1787761416.900198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874ac00.
00010000:00000010:2.1:1787761416.900200:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb900.
00010000:00000010:2.1:1787761416.900202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8770200.
00010000:00000010:2.1:1787761416.900204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27800.
00010000:00000010:2.1:1787761416.900206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa38400.
00010000:00000010:2.1:1787761416.900208:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb800.
00010000:00000010:2.1:1787761416.900210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8770e00.
00010000:00000010:2.1:1787761416.900212:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb500.
00010000:00000010:2.1:1787761416.900214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068b000.
00010000:00000010:2.1:1787761416.904178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4e00.
00010000:00000010:2.1:1787761416.904184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068a800.
00010000:00000010:2.1:1787761416.904187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fbe00.
00010000:00000010:2.1:1787761416.904189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068b600.
00010000:00000010:2.1:1787761416.904192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fbc00.
00010000:00000010:2.1:1787761416.904194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8771600.
00010000:00000010:2.1:1787761416.904196:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47d00.
00010000:00000010:2.1:1787761416.904198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e8000.
00010000:00000010:2.1:1787761416.904200:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27c00.
00010000:00000010:2.1:1787761416.904202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa38a00.
00010000:00000010:2.1:1787761416.904204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47200.
00010000:00000010:2.1:1787761416.904206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e9a00.
00010000:00000010:2.1:1787761416.904209:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fbb00.
00010000:00000010:2.1:1787761416.904211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068be00.
00010000:00000010:2.1:1787761416.904212:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb300.
00010000:00000010:2.1:1787761416.904214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068b400.
00010000:00000010:2.1:1787761416.904216:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47100.
00010000:00000010:2.1:1787761416.904218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068ae00.
00010000:00000010:2.1:1787761416.904220:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb600.
00010000:00000010:2.1:1787761416.904222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8771200.
00010000:00000010:2.1:1787761416.904224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fba00.
00010000:00000010:2.1:1787761416.904226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068a400.
00010000:00000010:2.1:1787761416.904227:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47900.
00010000:00000010:2.1:1787761416.904229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e9800.
00010000:00000010:2.1:1787761416.904231:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb000.
00010000:00000010:2.1:1787761416.904232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068b800.
00010000:00000010:2.1:1787761416.904234:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb200.
00010000:00000010:2.1:1787761416.904235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068a200.
00010000:00000010:2.1:1787761416.904237:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb400.
00010000:00000010:2.1:1787761416.904239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068ac00.
00010000:00000010:2.1:1787761416.904240:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fbd00.
00010000:00000010:2.1:1787761416.904242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8771e00.
00010000:00000010:2.1:1787761416.904243:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27b00.
00010000:00000010:2.1:1787761416.904245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa38c00.
00010000:00000010:2.1:1787761416.904247:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b300.
00010000:00000010:2.1:1787761416.904249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068a000.
00010000:00000010:2.1:1787761416.904250:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5000.
00010000:00000010:2.1:1787761416.904252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa39a00.
00010000:00000010:2.1:1787761416.904254:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b500.
00010000:00000010:2.1:1787761416.904256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8770800.
00010000:00000010:2.1:1787761416.904257:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47b00.
00010000:00000010:2.1:1787761416.904259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e9200.
00010000:00000010:2.1:1787761416.904261:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b100.
00010000:00000010:2.1:1787761416.904262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8770a00.
00010000:00000010:2.1:1787761416.904264:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b000.
00010000:00000010:2.1:1787761416.904265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068b200.
00010000:00000010:2.1:1787761416.904267:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841bb00.
00010000:00000010:2.1:1787761416.904268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e9600.
00010000:00000010:2.1:1787761416.904271:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5600.
00010000:00000010:2.1:1787761416.904272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa39e00.
00010000:00000010:2.1:1787761416.904274:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47700.
00010000:00000010:2.1:1787761416.904276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e9e00.
00010000:00000010:2.1:1787761416.904278:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f12a5d00.
00010000:00000010:2.1:1787761416.904279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa39800.
00010000:00000010:2.1:1787761416.904281:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b400.
00010000:00000010:2.1:1787761416.904283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa39000.
00010000:00000010:2.1:1787761416.904284:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47e00.
00010000:00000010:2.1:1787761416.904286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e8a00.
00010000:00000010:2.1:1787761416.904287:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841b900.
00010000:00000010:2.1:1787761416.904289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa38200.
00010000:00000010:2.1:1787761416.904290:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47c00.
00010000:00000010:2.1:1787761416.904292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e9400.
00010000:00000010:2.1:1787761416.904294:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47400.
00010000:00000010:2.1:1787761416.904295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068ba00.
00010000:00000010:2.1:1787761416.904297:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841bd00.
00010000:00000010:2.1:1787761416.904298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d783a600.
00010000:00000010:2.1:1787761416.904313:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4800.
00010000:00000010:2.1:1787761416.904315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8771c00.
00010000:00000010:2.1:1787761416.904317:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd47000.
00010000:00000010:2.1:1787761416.904318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84e8e00.
00010000:00000010:2.1:1787761416.904320:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4300.
00010000:00000010:2.1:1787761416.904321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8771400.
00010000:00000010:2.1:1787761416.904323:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f841bf00.
00010000:00000010:2.1:1787761416.904324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa38800.
00000800:00000200:1.1:1787761418.300765:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761418.300772:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 9 secs later)
00000800:00000200:1.1:1787761418.301317:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761418.301318:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 9 secs later)
00000800:00000200:1.1:1787761418.301517:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761418.301518:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 9 secs later)
00000800:00000200:1.1:1787761418.301718:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761418.301720:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 9 secs later)
00000800:00000200:1.1:1787761423.421021:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761423.421029:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 4 secs later)
00000800:00000200:1.1:1787761423.421784:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761423.421787:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 4 secs later)
00000800:00000200:1.1:1787761423.422032:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761423.422034:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 4 secs later)
00000800:00000200:1.1:1787761423.422255:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761423.422258:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 32, 4 secs later)
00010000:00000010:2.1:1787761427.140176:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858ce00.
00010000:00000010:2.1:1787761427.140185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7899800.
00010000:00000010:2.1:1787761427.140190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565c00.
00010000:00000010:2.1:1787761427.140194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787b400.
00010000:00000010:2.1:1787761427.140199:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2d600.
00010000:00000010:2.1:1787761427.140202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a1600.
00010000:00000010:2.1:1787761427.140206:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2d400.
00010000:00000010:2.1:1787761427.140209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a1200.
00010000:00000010:2.1:1787761427.140212:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c400.
00010000:00000010:2.1:1787761427.140215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874a800.
00010000:00000010:2.1:1787761427.140219:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2dc00.
00010000:00000010:2.1:1787761427.140222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0600.
00010000:00000010:2.1:1787761427.140225:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35db00.
00010000:00000010:2.1:1787761427.140228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787ae00.
00010000:00000010:2.1:1787761427.140231:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2d500.
00010000:00000010:2.1:1787761427.140234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d6c00.
00010000:00000010:2.1:1787761427.140236:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2d000.
00010000:00000010:2.1:1787761427.140239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d7e00.
00010000:00000010:2.1:1787761427.140242:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c100.
00010000:00000010:2.1:1787761427.140245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d7200.
00010000:00000010:2.1:1787761427.140248:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35d700.
00010000:00000010:2.1:1787761427.140252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874b600.
00010000:00000010:2.1:1787761427.140255:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c300.
00010000:00000010:2.1:1787761427.140258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874b400.
00010000:00000010:2.1:1787761427.140261:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c800.
00010000:00000010:2.1:1787761427.140263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874a600.
00010000:00000010:2.1:1787761427.140265:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858cf00.
00010000:00000010:2.1:1787761427.140267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787be00.
00010000:00000010:2.1:1787761427.140269:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2d800.
00010000:00000010:2.1:1787761427.140271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787bc00.
00010000:00000010:2.1:1787761427.140273:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27600.
00010000:00000010:2.1:1787761427.140275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787aa00.
00010000:00000010:2.1:1787761427.140276:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2d300.
00010000:00000010:2.1:1787761427.140278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d7c00.
00010000:00000010:2.1:1787761427.140280:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858cc00.
00010000:00000010:2.1:1787761427.140282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874bc00.
00010000:00000010:2.1:1787761427.140284:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35de00.
00010000:00000010:2.1:1787761427.140286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787b200.
00010000:00000010:2.1:1787761427.140288:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35d600.
00010000:00000010:2.1:1787761427.140290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787ba00.
00010000:00000010:2.1:1787761427.140291:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2da00.
00010000:00000010:2.1:1787761427.140293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0e00.
00010000:00000010:2.1:1787761427.140295:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858cb00.
00010000:00000010:2.1:1787761427.140297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874be00.
00010000:00000010:2.1:1787761427.140298:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2d700.
00010000:00000010:2.1:1787761427.140300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787b600.
00010000:00000010:2.1:1787761427.140301:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2d200.
00010000:00000010:2.1:1787761427.140303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8771800.
00010000:00000010:2.1:1787761427.140305:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2d100.
00010000:00000010:2.1:1787761427.140307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8771a00.
00010000:00000010:2.1:1787761427.140308:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2d900.
00010000:00000010:2.1:1787761427.140310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068bc00.
00010000:00000010:2.1:1787761427.140311:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2dd00.
00010000:00000010:2.1:1787761427.140313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa39600.
00010000:00000010:2.1:1787761427.140315:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35d800.
00010000:00000010:2.1:1787761427.140317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8770c00.
00010000:00000010:2.1:1787761427.144173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35d900.
00010000:00000010:2.1:1787761427.144180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8770000.
00010000:00000010:2.1:1787761427.144184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2df00.
00010000:00000010:2.1:1787761427.144187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa38000.
00010000:00000010:2.1:1787761427.144189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2de00.
00010000:00000010:2.1:1787761427.144192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa38e00.
00010000:00000010:2.1:1787761427.144195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858ca00.
00010000:00000010:2.1:1787761427.144197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874a000.
00010000:00000010:2.1:1787761427.144201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35dd00.
00010000:00000010:2.1:1787761427.144203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa39400.
00010000:00000010:2.1:1787761427.144206:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c700.
00010000:00000010:2.1:1787761427.144209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874b200.
00010000:00000010:2.1:1787761427.144212:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c000.
00010000:00000010:2.1:1787761427.144215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874b000.
00010000:00000010:2.1:1787761427.144218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c200.
00010000:00000010:2.1:1787761427.144221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874ba00.
00010000:00000010:2.1:1787761427.144224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0e2db00.
00010000:00000010:2.1:1787761427.144226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8771000.
00010000:00000010:2.1:1787761427.144229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751daee4a00.
00010000:00000010:2.1:1787761427.144232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874ae00.
00010000:00000010:2.1:1787761427.144235:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fb700.
00010000:00000010:2.1:1787761427.144238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8770400.
00010000:00000010:2.1:1787761427.144241:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c900.
00010000:00000010:2.1:1787761427.144244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874a400.
00010000:00000010:2.1:1787761427.144247:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27f00.
00010000:00000010:2.1:1787761427.144250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa39200.
00010000:00000010:2.1:1787761427.144253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78fbf00.
00010000:00000010:2.1:1787761427.144255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff97520068a600.
00010000:00000010:2.1:1787761427.144258:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858cd00.
00010000:00000010:2.1:1787761427.144261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874b800.
00010000:00000010:2.1:1787761427.144264:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27400.
00010000:00000010:2.1:1787761427.144266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f874a200.
00000800:00000200:1.1:1787761428.540782:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761428.541166:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761428.541313:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761428.541468:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761433.660852:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761433.660857:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787761433.661624:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761433.661627:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787761433.661899:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761433.661901:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787761433.662099:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761433.662101:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00010000:00000010:2.1:1787761436.356166:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46b00.
00010000:00000010:2.1:1787761436.356173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d6000.
00010000:00000010:2.1:1787761436.356177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22100.
00010000:00000010:2.1:1787761436.356179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7899e00.
00010000:00000010:2.1:1787761436.356182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22900.
00010000:00000010:2.1:1787761436.356184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7898000.
00010000:00000010:2.1:1787761436.356186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30500.
00010000:00000010:2.1:1787761436.356188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d7800.
00010000:00000010:2.1:1787761436.356190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22500.
00010000:00000010:2.1:1787761436.356192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7899200.
00010000:00000010:2.1:1787761436.356194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565a00.
00010000:00000010:2.1:1787761436.356196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a1400.
00010000:00000010:2.1:1787761436.356198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22e00.
00010000:00000010:2.1:1787761436.356200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7898800.
00010000:00000010:2.1:1787761436.356202:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35df00.
00010000:00000010:2.1:1787761436.356204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d6400.
00010000:00000010:2.1:1787761436.356206:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35dc00.
00010000:00000010:2.1:1787761436.356208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa435c00.
00010000:00000010:2.1:1787761436.356210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35da00.
00010000:00000010:2.1:1787761436.356212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a1000.
00010000:00000010:2.1:1787761436.356214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565b00.
00010000:00000010:2.1:1787761436.356215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787b800.
00010000:00000010:2.1:1787761436.356217:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565800.
00010000:00000010:2.1:1787761436.356219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0c00.
00010000:00000010:2.1:1787761436.356221:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565300.
00010000:00000010:2.1:1787761436.356223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787a800.
00010000:00000010:2.1:1787761436.356224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35d300.
00010000:00000010:2.1:1787761436.356226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a1800.
00010000:00000010:2.1:1787761436.356227:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565700.
00010000:00000010:2.1:1787761436.356229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0200.
00010000:00000010:2.1:1787761436.356231:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565000.
00010000:00000010:2.1:1787761436.356233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787a200.
00010000:00000010:2.1:1787761436.356235:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565200.
00010000:00000010:2.1:1787761436.356236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787a000.
00010000:00000010:2.1:1787761436.356239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22b00.
00010000:00000010:2.1:1787761436.356240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7899000.
00010000:00000010:2.1:1787761436.356242:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27a00.
00010000:00000010:2.1:1787761436.356244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d6800.
00010000:00000010:2.1:1787761436.356245:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565100.
00010000:00000010:2.1:1787761436.356247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787b000.
00010000:00000010:2.1:1787761436.356249:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565400.
00010000:00000010:2.1:1787761436.356250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787a600.
00010000:00000010:2.1:1787761436.356252:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27d00.
00010000:00000010:2.1:1787761436.356253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787ac00.
00010000:00000010:2.1:1787761436.356255:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565900.
00010000:00000010:2.1:1787761436.356257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0a00.
00010000:00000010:2.1:1787761436.356259:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565d00.
00010000:00000010:2.1:1787761436.356261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d6600.
00010000:00000010:2.1:1787761436.356262:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22300.
00010000:00000010:2.1:1787761436.356264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7898e00.
00010000:00000010:2.1:1787761436.356266:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35d500.
00010000:00000010:2.1:1787761436.356268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a1e00.
00010000:00000010:2.1:1787761436.356269:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27900.
00010000:00000010:2.1:1787761436.356271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d7000.
00010000:00000010:2.1:1787761436.356272:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27500.
00010000:00000010:2.1:1787761436.356274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d6e00.
00010000:00000010:2.1:1787761436.356276:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565f00.
00010000:00000010:2.1:1787761436.356277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a1c00.
00010000:00000010:2.1:1787761436.356279:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f858c600.
00010000:00000010:2.1:1787761436.356280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7898400.
00010000:00000010:2.1:1787761436.356282:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27300.
00010000:00000010:2.1:1787761436.356284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d7400.
00010000:00000010:2.1:1787761436.356285:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ca35d100.
00010000:00000010:2.1:1787761436.356287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d787a400.
00010000:00000010:2.1:1787761436.356288:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565e00.
00010000:00000010:2.1:1787761436.356290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0000.
00010000:00000010:2.1:1787761436.356291:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1d27e00.
00010000:00000010:2.1:1787761436.356293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d6a00.
00000800:00000200:0.1:1787761438.781147:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761438.781171:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787761438.781778:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761438.781781:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787761438.782182:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761438.782184:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787761438.782478:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761438.782483:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787761443.900765:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761443.900769:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787761443.901257:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761443.901259:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787761443.901445:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761443.901447:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787761443.901658:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761443.901660:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00010000:00000010:0.1:1787761446.597172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6a400.
00010000:00000010:0.1:1787761446.597180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867b000.
00010000:00000010:0.1:1787761446.597185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22000.
00010000:00000010:0.1:1787761446.597189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa52400.
00010000:00000010:0.1:1787761446.597192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30300.
00010000:00000010:0.1:1787761446.597195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867a800.
00010000:00000010:0.1:1787761446.597198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6ad00.
00010000:00000010:0.1:1787761446.597201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434600.
00010000:00000010:0.1:1787761446.597205:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6a600.
00010000:00000010:0.1:1787761446.597207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53800.
00010000:00000010:0.1:1787761446.597210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6ab00.
00010000:00000010:0.1:1787761446.597213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867ba00.
00010000:00000010:0.1:1787761446.597216:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6ae00.
00010000:00000010:0.1:1787761446.597219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79ab400.
00010000:00000010:0.1:1787761446.597222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46700.
00010000:00000010:0.1:1787761446.597225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434400.
00010000:00000010:0.1:1787761446.597228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6af00.
00010000:00000010:0.1:1787761446.597230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867ae00.
00010000:00000010:0.1:1787761446.597233:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46100.
00010000:00000010:0.1:1787761446.597236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434e00.
00010000:00000010:0.1:1787761446.597239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6ac00.
00010000:00000010:0.1:1787761446.597242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d7600.
00010000:00000010:0.1:1787761446.597244:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46400.
00010000:00000010:0.1:1787761446.597247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867a400.
00010000:00000010:0.1:1787761446.597250:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46d00.
00010000:00000010:0.1:1787761446.597252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa435200.
00010000:00000010:0.1:1787761446.597255:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46500.
00010000:00000010:0.1:1787761446.597258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa435600.
00010000:00000010:0.1:1787761446.597260:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22c00.
00010000:00000010:0.1:1787761446.597264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7899600.
00010000:00000010:0.1:1787761446.597266:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22600.
00010000:00000010:0.1:1787761446.597269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7898c00.
00010000:00000010:0.1:1787761446.597272:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22800.
00010000:00000010:0.1:1787761446.597276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7899c00.
00010000:00000010:0.1:1787761446.597278:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22f00.
00010000:00000010:0.1:1787761446.597281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7899a00.
00010000:00000010:0.1:1787761446.597284:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6a300.
00010000:00000010:0.1:1787761446.597286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867a200.
00010000:00000010:0.1:1787761446.597289:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6aa00.
00010000:00000010:0.1:1787761446.597292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867ac00.
00010000:00000010:0.1:1787761446.597295:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6a700.
00010000:00000010:0.1:1787761446.597298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867be00.
00010000:00000010:0.1:1787761446.597300:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22d00.
00010000:00000010:0.1:1787761446.597303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7898200.
00010000:00000010:0.1:1787761446.597305:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22200.
00010000:00000010:0.1:1787761446.597308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7898a00.
00010000:00000010:0.1:1787761446.597311:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22a00.
00010000:00000010:0.1:1787761446.597313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7899400.
00010000:00000010:0.1:1787761446.597316:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6a000.
00010000:00000010:0.1:1787761446.597319:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867a000.
00010000:00000010:0.1:1787761446.597322:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6a200.
00010000:00000010:0.1:1787761446.597325:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867b600.
00010000:00000010:0.1:1787761446.597328:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6a100.
00010000:00000010:0.1:1787761446.597331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867b200.
00010000:00000010:0.1:1787761446.597333:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46c00.
00010000:00000010:0.1:1787761446.597335:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d7a00.
00010000:00000010:0.1:1787761446.597338:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22700.
00010000:00000010:0.1:1787761446.597340:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7898600.
00010000:00000010:0.1:1787761446.597343:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30600.
00010000:00000010:0.1:1787761446.597345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84d6200.
00010000:00000010:0.1:1787761446.597348:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6a900.
00010000:00000010:0.1:1787761446.597351:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0800.
00010000:00000010:0.1:1787761446.597353:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30800.
00010000:00000010:0.1:1787761446.597356:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa435800.
00010000:00000010:0.1:1787761446.597358:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6a500.
00010000:00000010:0.1:1787761446.597362:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a1a00.
00010000:00000010:0.1:1787761446.597364:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565500.
00010000:00000010:0.1:1787761446.597367:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa435000.
00010000:00000010:0.1:1787761446.597370:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c4565600.
00010000:00000010:0.1:1787761446.597373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f59a0400.
00010000:00000010:0.1:1787761446.597375:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30d00.
00010000:00000010:0.1:1787761446.597378:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434200.
00000800:00000200:1.1:1787761449.020700:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761449.020706:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787761449.021210:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761449.021212:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787761449.021402:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761449.021404:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787761449.021576:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761449.021578:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787761454.140934:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761454.140946:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787761454.141819:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761454.141823:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787761454.142054:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761454.142056:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787761454.142296:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761454.142298:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00010000:00000010:2.1:1787761456.835167:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46200.
00010000:00000010:2.1:1787761456.835174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8592800.
00010000:00000010:2.1:1787761456.835177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784e700.
00010000:00000010:2.1:1787761456.835204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa52a00.
00010000:00000010:2.1:1787761456.835207:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2a00.
00010000:00000010:2.1:1787761456.835209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8593800.
00010000:00000010:2.1:1787761456.835211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46000.
00010000:00000010:2.1:1787761456.835213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8592200.
00010000:00000010:2.1:1787761456.835215:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784e200.
00010000:00000010:2.1:1787761456.835217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53e00.
00010000:00000010:2.1:1787761456.835220:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784e100.
00010000:00000010:2.1:1787761456.835222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53c00.
00010000:00000010:2.1:1787761456.835224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2600.
00010000:00000010:2.1:1787761456.835226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8761600.
00010000:00000010:2.1:1787761456.835228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46600.
00010000:00000010:2.1:1787761456.835230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434a00.
00010000:00000010:2.1:1787761456.839166:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46300.
00010000:00000010:2.1:1787761456.839172:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8761200.
00010000:00000010:2.1:1787761456.839174:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46800.
00010000:00000010:2.1:1787761456.839176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79ab200.
00010000:00000010:2.1:1787761456.839179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2200.
00010000:00000010:2.1:1787761456.839181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434c00.
00010000:00000010:2.1:1787761456.839182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2100.
00010000:00000010:2.1:1787761456.839184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867bc00.
00010000:00000010:2.1:1787761456.839187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784e900.
00010000:00000010:2.1:1787761456.839189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa52000.
00010000:00000010:2.1:1787761456.839191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30000.
00010000:00000010:2.1:1787761456.839193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79ab600.
00010000:00000010:2.1:1787761456.839195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2500.
00010000:00000010:2.1:1787761456.839196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867b400.
00010000:00000010:2.1:1787761456.839198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784ed00.
00010000:00000010:2.1:1787761456.839200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53600.
00010000:00000010:2.1:1787761456.839202:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30900.
00010000:00000010:2.1:1787761456.839203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa435e00.
00010000:00000010:2.1:1787761456.839205:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2e00.
00010000:00000010:2.1:1787761456.839207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867a600.
00010000:00000010:2.1:1787761456.839208:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2b00.
00010000:00000010:2.1:1787761456.839210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa435400.
00010000:00000010:2.1:1787761456.839212:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784e500.
00010000:00000010:2.1:1787761456.839213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867aa00.
00010000:00000010:2.1:1787761456.839215:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784ee00.
00010000:00000010:2.1:1787761456.839216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434800.
00010000:00000010:2.1:1787761456.839218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2800.
00010000:00000010:2.1:1787761456.839220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867b800.
00010000:00000010:2.1:1787761456.839222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46a00.
00010000:00000010:2.1:1787761456.839224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aa600.
00010000:00000010:2.1:1787761456.839226:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6a800.
00010000:00000010:2.1:1787761456.839228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa435a00.
00010000:00000010:2.1:1787761456.839229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784ec00.
00010000:00000010:2.1:1787761456.839231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa52600.
00010000:00000010:2.1:1787761456.839232:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30c00.
00010000:00000010:2.1:1787761456.839234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79ab000.
00010000:00000010:2.1:1787761456.839236:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784e800.
00010000:00000010:2.1:1787761456.839237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53000.
00010000:00000010:2.1:1787761456.839239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb22400.
00010000:00000010:2.1:1787761456.839245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa52e00.
00010000:00000010:2.1:1787761456.839247:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30200.
00010000:00000010:2.1:1787761456.839248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aa400.
00000800:00000200:1.1:1787761459.260824:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761459.260829:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787761459.261345:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761459.261347:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787761459.261550:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761459.261551:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787761459.261731:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761459.261733:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787761464.380786:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761464.380793:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787761464.381368:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761464.381370:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787761464.381575:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761464.381577:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787761464.381783:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761464.381786:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00010000:00000010:1.1:1787761467.075158:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71900.
00010000:00000010:1.1:1787761467.075166:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8593a00.
00010000:00000010:1.1:1787761467.075169:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30400.
00010000:00000010:1.1:1787761467.075171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8783a00.
00010000:00000010:1.1:1787761467.075173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150300.
00010000:00000010:1.1:1787761467.075176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8761400.
00010000:00000010:1.1:1787761467.075178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46900.
00010000:00000010:1.1:1787761467.075180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8592a00.
00010000:00000010:1.1:1787761467.075182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150600.
00010000:00000010:1.1:1787761467.075184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8760e00.
00010000:00000010:1.1:1787761467.075186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71500.
00010000:00000010:1.1:1787761467.075188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8760a00.
00010000:00000010:1.1:1787761467.075190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71e00.
00010000:00000010:1.1:1787761467.075191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8760600.
00010000:00000010:1.1:1787761467.075193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71b00.
00010000:00000010:1.1:1787761467.075195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8760000.
00010000:00000010:1.1:1787761467.079169:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150700.
00010000:00000010:1.1:1787761467.079175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8761e00.
00010000:00000010:1.1:1787761467.079177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71300.
00010000:00000010:1.1:1787761467.079179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8760200.
00010000:00000010:1.1:1787761467.079181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71a00.
00010000:00000010:1.1:1787761467.079183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8592400.
00010000:00000010:1.1:1787761467.079185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2400.
00010000:00000010:1.1:1787761467.079187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8593400.
00010000:00000010:1.1:1787761467.079189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2f00.
00010000:00000010:1.1:1787761467.079191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8761000.
00010000:00000010:1.1:1787761467.079193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784ea00.
00010000:00000010:1.1:1787761467.079195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8760800.
00010000:00000010:1.1:1787761467.079197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784ef00.
00010000:00000010:1.1:1787761467.079199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53400.
00010000:00000010:1.1:1787761467.079202:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784e300.
00010000:00000010:1.1:1787761467.079204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53a00.
00010000:00000010:1.1:1787761467.079206:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2300.
00010000:00000010:1.1:1787761467.079208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8761a00.
00010000:00000010:1.1:1787761467.079210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2700.
00010000:00000010:1.1:1787761467.079212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aa200.
00010000:00000010:1.1:1787761467.079214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2d00.
00010000:00000010:1.1:1787761467.079216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8760400.
00010000:00000010:1.1:1787761467.079218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2c00.
00010000:00000010:1.1:1787761467.079221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8761800.
00010000:00000010:1.1:1787761467.079224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784eb00.
00010000:00000010:1.1:1787761467.079226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa52800.
00010000:00000010:1.1:1787761467.079229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30e00.
00010000:00000010:1.1:1787761467.079232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79abe00.
00010000:00000010:1.1:1787761467.079235:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30a00.
00010000:00000010:1.1:1787761467.079237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aa000.
00010000:00000010:1.1:1787761467.079240:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784e400.
00010000:00000010:1.1:1787761467.079243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8592000.
00010000:00000010:1.1:1787761467.079246:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784e000.
00010000:00000010:1.1:1787761467.079248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa52c00.
00010000:00000010:1.1:1787761467.079251:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2900.
00010000:00000010:1.1:1787761467.079254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8760c00.
00010000:00000010:1.1:1787761467.079257:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d784e600.
00010000:00000010:1.1:1787761467.079260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa52200.
00010000:00000010:1.1:1787761467.079262:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a2000.
00010000:00000010:1.1:1787761467.079265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8761c00.
00010000:00000010:1.1:1787761467.079268:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46e00.
00010000:00000010:1.1:1787761467.079271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8593c00.
00010000:00000010:1.1:1787761467.079273:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfd46f00.
00010000:00000010:1.1:1787761467.079285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8593200.
00000800:00000200:1.1:1787761469.500900:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761469.500908:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787761469.501541:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761469.501543:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787761469.501752:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761469.501754:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787761469.501932:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761469.501934:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787761474.620778:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761474.620783:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787761474.621257:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761474.621259:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787761474.621443:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761474.621445:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787761474.621622:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761474.621624:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00010000:00000010:0.1:1787761477.318175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150100.
00010000:00000010:0.1:1787761477.318182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85da000.
00010000:00000010:0.1:1787761477.318186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84ebe00.
00010000:00000010:0.1:1787761477.318188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa57200.
00010000:00000010:0.1:1787761477.318191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84ebb00.
00010000:00000010:0.1:1787761477.318193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa57a00.
00010000:00000010:0.1:1787761477.318195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71800.
00010000:00000010:0.1:1787761477.318198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8782400.
00010000:00000010:0.1:1787761477.318200:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84eb300.
00010000:00000010:0.1:1787761477.318202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa56a00.
00010000:00000010:0.1:1787761477.318204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1e29700.
00010000:00000010:0.1:1787761477.318205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85da600.
00010000:00000010:0.1:1787761477.318208:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30100.
00010000:00000010:0.1:1787761477.318210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aa800.
00010000:00000010:0.1:1787761477.318213:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71f00.
00010000:00000010:0.1:1787761477.318215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8782200.
00010000:00000010:0.1:1787761477.318217:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150d00.
00010000:00000010:0.1:1787761477.318219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85db000.
00010000:00000010:0.1:1787761477.318221:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84eba00.
00010000:00000010:0.1:1787761477.318223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8593600.
00010000:00000010:0.1:1787761477.318225:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71200.
00010000:00000010:0.1:1787761477.318227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8782a00.
00010000:00000010:0.1:1787761477.318228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30b00.
00010000:00000010:0.1:1787761477.318230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8592600.
00010000:00000010:0.1:1787761477.318232:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71700.
00010000:00000010:0.1:1787761477.318233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8593000.
00010000:00000010:0.1:1787761477.318235:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150500.
00010000:00000010:0.1:1787761477.318237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85dae00.
00010000:00000010:0.1:1787761477.318238:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30700.
00010000:00000010:0.1:1787761477.318240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85db400.
00010000:00000010:0.1:1787761477.318241:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71600.
00010000:00000010:0.1:1787761477.318243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8592c00.
00010000:00000010:0.1:1787761477.318244:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71c00.
00010000:00000010:0.1:1787761477.318246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8782000.
00010000:00000010:0.1:1787761477.318247:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71d00.
00010000:00000010:0.1:1787761477.318249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8592e00.
00010000:00000010:0.1:1787761477.318250:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71100.
00010000:00000010:0.1:1787761477.318252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8593e00.
00010000:00000010:0.1:1787761477.318254:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150e00.
00010000:00000010:0.1:1787761477.318255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85da400.
00010000:00000010:0.1:1787761477.318257:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71000.
00010000:00000010:0.1:1787761477.318258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8782600.
00010000:00000010:0.1:1787761477.318260:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150b00.
00010000:00000010:0.1:1787761477.318261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa53200.
00010000:00000010:0.1:1787761477.318263:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d30f00.
00010000:00000010:0.1:1787761477.318289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79ab800.
00010000:00000010:0.1:1787761477.318291:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4e71400.
00010000:00000010:0.1:1787761477.318294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8783000.
00000800:00000200:1.1:1787761479.741027:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761479.741045:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787761479.742527:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761479.742530:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787761479.742916:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761479.742919:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787761479.743393:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761479.743396:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787761484.861184:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761484.861192:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787761484.862194:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761484.862197:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787761484.862537:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761484.862540:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787761484.862931:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761484.862934:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00010000:00000010:0.1:1787761487.556167:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467af00.
00010000:00000010:0.1:1787761487.556176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85dba00.
00010000:00000010:0.1:1787761487.556179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150400.
00010000:00000010:0.1:1787761487.556182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85da800.
00010000:00000010:0.1:1787761487.556184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467a500.
00010000:00000010:0.1:1787761487.556186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8782c00.
00010000:00000010:0.1:1787761487.556188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467ad00.
00010000:00000010:0.1:1787761487.556190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8783e00.
00010000:00000010:0.1:1787761487.556193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84ebc00.
00010000:00000010:0.1:1787761487.556195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa56000.
00010000:00000010:0.1:1787761487.556197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84eb700.
00010000:00000010:0.1:1787761487.556199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa57600.
00010000:00000010:0.1:1787761487.556201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150f00.
00010000:00000010:0.1:1787761487.556203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85db200.
00010000:00000010:0.1:1787761487.556206:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84eb800.
00010000:00000010:0.1:1787761487.556207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa56200.
00010000:00000010:0.1:1787761487.556209:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84ebf00.
00010000:00000010:0.1:1787761487.556210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aba00.
00010000:00000010:0.1:1787761487.556212:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150a00.
00010000:00000010:0.1:1787761487.556214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85dbc00.
00010000:00000010:0.1:1787761487.556216:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150000.
00010000:00000010:0.1:1787761487.556217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85dbe00.
00010000:00000010:0.1:1787761487.556219:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1e29000.
00010000:00000010:0.1:1787761487.556221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa56600.
00010000:00000010:0.1:1787761487.556222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84eb000.
00010000:00000010:0.1:1787761487.556224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa57c00.
00010000:00000010:0.1:1787761487.556225:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1e29c00.
00010000:00000010:0.1:1787761487.556227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa57800.
00010000:00000010:0.1:1787761487.556229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84eb200.
00010000:00000010:0.1:1787761487.556230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa57000.
00010000:00000010:0.1:1787761487.556233:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467ab00.
00010000:00000010:0.1:1787761487.556235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8783200.
00010000:00000010:0.1:1787761487.556237:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467a800.
00010000:00000010:0.1:1787761487.556239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8783600.
00010000:00000010:0.1:1787761487.556241:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467a300.
00010000:00000010:0.1:1787761487.556243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8783c00.
00010000:00000010:0.1:1787761487.556244:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84eb100.
00010000:00000010:0.1:1787761487.556246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aae00.
00010000:00000010:0.1:1787761487.556248:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150c00.
00010000:00000010:0.1:1787761487.556249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85da200.
00010000:00000010:0.1:1787761487.556251:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467aa00.
00010000:00000010:0.1:1787761487.556252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa56400.
00010000:00000010:0.1:1787761487.556254:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84eb400.
00010000:00000010:0.1:1787761487.556255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa57e00.
00010000:00000010:0.1:1787761487.556257:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150900.
00010000:00000010:0.1:1787761487.556258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85dac00.
00010000:00000010:0.1:1787761487.556260:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467a000.
00010000:00000010:0.1:1787761487.556262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8783800.
00010000:00000010:0.1:1787761487.556263:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467a200.
00010000:00000010:0.1:1787761487.556265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa56c00.
00010000:00000010:0.1:1787761487.556266:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150200.
00010000:00000010:0.1:1787761487.556268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8782e00.
00010000:00000010:0.1:1787761487.556269:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1e29300.
00010000:00000010:0.1:1787761487.556271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa57400.
00010000:00000010:0.1:1787761487.556272:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84eb900.
00010000:00000010:0.1:1787761487.556274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79abc00.
00010000:00000010:0.1:1787761487.556275:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1e29200.
00010000:00000010:0.1:1787761487.556277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aac00.
00010000:00000010:0.1:1787761487.556279:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467a400.
00010000:00000010:0.1:1787761487.556280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8783400.
00010000:00000010:0.1:1787761487.556282:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84eb500.
00010000:00000010:0.1:1787761487.556283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa56800.
00000800:00000200:1.1:1787761489.980889:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761489.980895:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787761489.981426:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761489.981428:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787761489.981655:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761489.981656:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787761489.981874:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761489.981876:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787761495.101454:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761495.101956:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761495.102291:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761495.102628:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00010000:00000010:2.1:1787761497.795176:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635d00.
00010000:00000010:2.1:1787761497.795184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870fe00.
00010000:00000010:2.1:1787761497.795188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039c00.
00010000:00000010:2.1:1787761497.795192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8622400.
00010000:00000010:2.1:1787761497.795248:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635e00.
00010000:00000010:2.1:1787761497.795250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85db800.
00010000:00000010:2.1:1787761497.795253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039d00.
00010000:00000010:2.1:1787761497.795256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8622200.
00010000:00000010:2.1:1787761497.795257:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039100.
00010000:00000010:2.1:1787761497.795260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870ec00.
00010000:00000010:2.1:1787761497.795261:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635b00.
00010000:00000010:2.1:1787761497.795263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85db600.
00010000:00000010:2.1:1787761497.795265:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518d600.
00010000:00000010:2.1:1787761497.795267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3ca00.
00010000:00000010:2.1:1787761497.795269:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039900.
00010000:00000010:2.1:1787761497.795271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8623400.
00010000:00000010:2.1:1787761497.795274:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039500.
00010000:00000010:2.1:1787761497.795276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870e000.
00010000:00000010:2.1:1787761497.799174:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039e00.
00010000:00000010:2.1:1787761497.799180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870f600.
00010000:00000010:2.1:1787761497.799182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039b00.
00010000:00000010:2.1:1787761497.799185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870e600.
00010000:00000010:2.1:1787761497.799187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039300.
00010000:00000010:2.1:1787761497.799189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8623e00.
00010000:00000010:2.1:1787761497.799191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039600.
00010000:00000010:2.1:1787761497.799193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8622000.
00010000:00000010:2.1:1787761497.799194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039a00.
00010000:00000010:2.1:1787761497.799196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870fa00.
00010000:00000010:2.1:1787761497.799198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84ebd00.
00010000:00000010:2.1:1787761497.799200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8623600.
00010000:00000010:2.1:1787761497.799202:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4150800.
00010000:00000010:2.1:1787761497.799205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85daa00.
00010000:00000010:2.1:1787761497.799207:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039200.
00010000:00000010:2.1:1787761497.799209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8622c00.
00010000:00000010:2.1:1787761497.799210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467ae00.
00010000:00000010:2.1:1787761497.799212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8622a00.
00010000:00000010:2.1:1787761497.799214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1e29e00.
00010000:00000010:2.1:1787761497.799216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfa56e00.
00010000:00000010:2.1:1787761497.799218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467a100.
00010000:00000010:2.1:1787761497.799220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8622e00.
00010000:00000010:2.1:1787761497.799222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467a900.
00010000:00000010:2.1:1787761497.799224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870e400.
00010000:00000010:2.1:1787761497.799226:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f84eb600.
00010000:00000010:2.1:1787761497.799228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fa434000.
00010000:00000010:2.1:1787761497.799235:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467a700.
00010000:00000010:2.1:1787761497.799237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870f800.
00010000:00000010:2.1:1787761497.799239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467a600.
00010000:00000010:2.1:1787761497.799240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870e200.
00010000:00000010:2.1:1787761497.799243:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f467ac00.
00010000:00000010:2.1:1787761497.799245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8782800.
00010000:00000010:2.1:1787761497.799247:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d1e29800.
00010000:00000010:2.1:1787761497.799249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aaa00.
00000800:00000200:1.1:1787761500.220924:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761500.220934:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787761500.221723:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761500.221725:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787761500.221987:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761500.221988:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787761500.222217:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761500.222218:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787761505.340934:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761505.340940:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1787761505.341418:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761505.341420:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1787761505.341678:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761505.341681:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1787761505.341929:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761505.341931:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00010000:00000010:2.1:1787761508.035180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748000.
00010000:00000010:2.1:1787761508.035189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479e000.
00010000:00000010:2.1:1787761508.035192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518d000.
00010000:00000010:2.1:1787761508.035195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3da00.
00010000:00000010:2.1:1787761508.035197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748400.
00010000:00000010:2.1:1787761508.035199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8622800.
00010000:00000010:2.1:1787761508.035201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748900.
00010000:00000010:2.1:1787761508.035203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8623000.
00010000:00000010:2.1:1787761508.035205:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748500.
00010000:00000010:2.1:1787761508.035206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479f000.
00010000:00000010:2.1:1787761508.035208:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635600.
00010000:00000010:2.1:1787761508.035210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479fa00.
00010000:00000010:2.1:1787761508.035211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748f00.
00010000:00000010:2.1:1787761508.035213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479ee00.
00010000:00000010:2.1:1787761508.035215:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873dc00.
00010000:00000010:2.1:1787761508.035217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8623a00.
00010000:00000010:2.1:1787761508.035220:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039700.
00010000:00000010:2.1:1787761508.035222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870f400.
00010000:00000010:2.1:1787761508.039178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635300.
00010000:00000010:2.1:1787761508.039184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8623200.
00010000:00000010:2.1:1787761508.039187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039000.
00010000:00000010:2.1:1787761508.039189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870ea00.
00010000:00000010:2.1:1787761508.039191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635a00.
00010000:00000010:2.1:1787761508.039194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980000.
00010000:00000010:2.1:1787761508.039196:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039400.
00010000:00000010:2.1:1787761508.039198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8623c00.
00010000:00000010:2.1:1787761508.039200:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635200.
00010000:00000010:2.1:1787761508.039202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8623800.
00010000:00000010:2.1:1787761508.039206:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039f00.
00010000:00000010:2.1:1787761508.039208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870ee00.
00010000:00000010:2.1:1787761508.039210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873db00.
00010000:00000010:2.1:1787761508.039212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870f000.
00010000:00000010:2.1:1787761508.039214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873d800.
00010000:00000010:2.1:1787761508.039216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870e800.
00010000:00000010:2.1:1787761508.039218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518d200.
00010000:00000010:2.1:1787761508.039220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870f200.
00010000:00000010:2.1:1787761508.039221:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518d100.
00010000:00000010:2.1:1787761508.039223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3c400.
00010000:00000010:2.1:1787761508.039225:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635400.
00010000:00000010:2.1:1787761508.039227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980600.
00010000:00000010:2.1:1787761508.039228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518dc00.
00010000:00000010:2.1:1787761508.039230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3d800.
00010000:00000010:2.1:1787761508.039232:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff975201039800.
00010000:00000010:2.1:1787761508.039234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f870fc00.
00000800:00000200:1.1:1787761510.460854:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761510.460859:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787761510.461322:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761510.461324:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787761510.461616:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761510.461617:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787761510.461806:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761510.461808:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787761515.581106:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761515.581110:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787761515.582901:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761515.582911:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787761515.583693:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761515.583697:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787761515.584064:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761515.584067:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00010000:00000010:1.1:1787761518.275180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748b00.
00010000:00000010:1.1:1787761518.275188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8763000.
00010000:00000010:1.1:1787761518.275193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873d100.
00010000:00000010:1.1:1787761518.275197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8763800.
00010000:00000010:1.1:1787761518.275200:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748d00.
00010000:00000010:1.1:1787761518.275203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8762200.
00010000:00000010:1.1:1787761518.275206:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635500.
00010000:00000010:1.1:1787761518.275208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8763e00.
00010000:00000010:1.1:1787761518.275211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635800.
00010000:00000010:1.1:1787761518.275214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981c00.
00010000:00000010:1.1:1787761518.275218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635700.
00010000:00000010:1.1:1787761518.275221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980c00.
00010000:00000010:1.1:1787761518.275224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748100.
00010000:00000010:1.1:1787761518.275229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479ec00.
00010000:00000010:1.1:1787761518.275232:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748200.
00010000:00000010:1.1:1787761518.275235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479e200.
00010000:00000010:1.1:1787761518.279169:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873d900.
00010000:00000010:1.1:1787761518.279175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3c600.
00010000:00000010:1.1:1787761518.279178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748c00.
00010000:00000010:1.1:1787761518.279181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479e400.
00010000:00000010:1.1:1787761518.279183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748e00.
00010000:00000010:1.1:1787761518.279185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8763a00.
00010000:00000010:1.1:1787761518.279187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635100.
00010000:00000010:1.1:1787761518.279190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980200.
00010000:00000010:1.1:1787761518.279193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635c00.
00010000:00000010:1.1:1787761518.279195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981a00.
00010000:00000010:1.1:1787761518.279198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635f00.
00010000:00000010:1.1:1787761518.279201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980e00.
00010000:00000010:1.1:1787761518.279204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873d500.
00010000:00000010:1.1:1787761518.279206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981400.
00010000:00000010:1.1:1787761518.279209:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748800.
00010000:00000010:1.1:1787761518.279212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479ea00.
00010000:00000010:1.1:1787761518.279214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635900.
00010000:00000010:1.1:1787761518.279217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981800.
00010000:00000010:1.1:1787761518.279220:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748600.
00010000:00000010:1.1:1787761518.279223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8762800.
00010000:00000010:1.1:1787761518.279226:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518d400.
00010000:00000010:1.1:1787761518.279230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3d000.
00010000:00000010:1.1:1787761518.279233:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748a00.
00010000:00000010:1.1:1787761518.279237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479fc00.
00010000:00000010:1.1:1787761518.279240:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748700.
00010000:00000010:1.1:1787761518.279243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8762c00.
00000800:00000200:1.1:1787761520.701347:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761520.701363:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787761520.702427:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761520.702429:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787761520.702705:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761520.702707:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787761520.702923:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761520.702925:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787761525.821452:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761525.821461:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787761525.822409:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761525.822413:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787761525.822777:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761525.822780:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787761525.823122:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761525.823125:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00010000:00000010:0.1:1787761527.491160:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786f800.
00010000:00000010:0.1:1787761527.491166:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980400.
00010000:00000010:0.1:1787761527.491169:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873d700.
00010000:00000010:0.1:1787761527.491171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8762600.
00010000:00000010:0.1:1787761527.491173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518d800.
00010000:00000010:0.1:1787761527.491176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8763c00.
00010000:00000010:0.1:1787761527.491178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466600.
00010000:00000010:0.1:1787761527.491180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8763600.
00010000:00000010:0.1:1787761527.491182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786fa00.
00010000:00000010:0.1:1787761527.491184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8762000.
00010000:00000010:0.1:1787761527.491186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786f700.
00010000:00000010:0.1:1787761527.491188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981e00.
00010000:00000010:0.1:1787761527.491190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466700.
00010000:00000010:0.1:1787761527.491191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479fe00.
00010000:00000010:0.1:1787761527.491193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466200.
00010000:00000010:0.1:1787761527.491195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8762a00.
00010000:00000010:0.1:1787761527.491196:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466400.
00010000:00000010:0.1:1787761527.491198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8763400.
00010000:00000010:0.1:1787761527.491200:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786f000.
00010000:00000010:0.1:1787761527.491201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981600.
00010000:00000010:0.1:1787761527.491203:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518dd00.
00010000:00000010:0.1:1787761527.491204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3d600.
00010000:00000010:0.1:1787761527.491206:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466500.
00010000:00000010:0.1:1787761527.491208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8762400.
00010000:00000010:0.1:1787761527.491209:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466e00.
00010000:00000010:0.1:1787761527.491211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8763200.
00010000:00000010:0.1:1787761527.491212:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8748300.
00010000:00000010:0.1:1787761527.491215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479f200.
00010000:00000010:0.1:1787761527.495157:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786f200.
00010000:00000010:0.1:1787761527.495161:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980800.
00010000:00000010:0.1:1787761527.495163:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c7635000.
00010000:00000010:0.1:1787761527.495165:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479f600.
00000800:00000200:1.1:1787761530.940849:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761530.940853:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787761530.941391:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761530.941393:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787761530.941600:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761530.941602:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787761530.941779:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761530.941781:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787761536.061665:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761536.061679:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787761536.062838:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761536.062842:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787761536.063258:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761536.063262:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787761536.063706:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761536.063710:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00010000:00000010:2.1:1787761537.731169:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466f00.
00010000:00000010:2.1:1787761537.731178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d6600.
00010000:00000010:2.1:1787761537.731182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466c00.
00010000:00000010:2.1:1787761537.731186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876b800.
00010000:00000010:2.1:1787761537.731189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466d00.
00010000:00000010:2.1:1787761537.731192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831e00.
00010000:00000010:2.1:1787761537.731213:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786fe00.
00010000:00000010:2.1:1787761537.731217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876b400.
00010000:00000010:2.1:1787761537.731220:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786ff00.
00010000:00000010:2.1:1787761537.731222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d7a00.
00010000:00000010:2.1:1787761537.731224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786f400.
00010000:00000010:2.1:1787761537.731227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d6e00.
00010000:00000010:2.1:1787761537.731229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646100.
00010000:00000010:2.1:1787761537.731232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3cc00.
00010000:00000010:2.1:1787761537.731234:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786f900.
00010000:00000010:2.1:1787761537.731236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7980a00.
00010000:00000010:2.1:1787761537.731239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786fd00.
00010000:00000010:2.1:1787761537.731242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981200.
00010000:00000010:2.1:1787761537.731244:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466900.
00010000:00000010:2.1:1787761537.731246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479f400.
00010000:00000010:2.1:1787761537.731249:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786f500.
00010000:00000010:2.1:1787761537.731251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7981000.
00010000:00000010:2.1:1787761537.731253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466000.
00010000:00000010:2.1:1787761537.731255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8622600.
00010000:00000010:2.1:1787761537.731257:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466b00.
00010000:00000010:2.1:1787761537.731259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479f800.
00010000:00000010:2.1:1787761537.731261:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466800.
00010000:00000010:2.1:1787761537.731263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479e600.
00010000:00000010:2.1:1787761537.731265:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466300.
00010000:00000010:2.1:1787761537.731267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c479e800.
00010000:00000010:2.1:1787761537.735160:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518d500.
00010000:00000010:2.1:1787761537.735167:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8762e00.
00000800:00000200:1.1:1787761541.180932:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761541.180938:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787761541.181472:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761541.181473:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787761541.181713:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761541.181716:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787761541.181925:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761541.181927:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787761546.301347:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761546.301359:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:1.1:1787761546.302407:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761546.302410:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:1.1:1787761546.302847:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761546.302851:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:1.1:1787761546.303254:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761546.303258:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00010000:00000010:0.1:1787761547.971160:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786fb00.
00010000:00000010:0.1:1787761547.971167:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d6c00.
00010000:00000010:0.1:1787761547.971170:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7db00.
00010000:00000010:0.1:1787761547.971173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876a400.
00010000:00000010:0.1:1787761547.971175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786f600.
00010000:00000010:0.1:1787761547.971177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d7e00.
00010000:00000010:0.1:1787761547.971179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873d000.
00010000:00000010:0.1:1787761547.971181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876aa00.
00010000:00000010:0.1:1787761547.971183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646b00.
00010000:00000010:0.1:1787761547.971185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876b200.
00010000:00000010:0.1:1787761547.971187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646c00.
00010000:00000010:0.1:1787761547.971188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3ce00.
00010000:00000010:0.1:1787761547.971191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873d300.
00010000:00000010:0.1:1787761547.971193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876bc00.
00010000:00000010:0.1:1787761547.971195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786f100.
00010000:00000010:0.1:1787761547.971196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d7c00.
00010000:00000010:0.1:1787761547.971198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873de00.
00010000:00000010:0.1:1787761547.971200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876ae00.
00010000:00000010:0.1:1787761547.971201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873dd00.
00010000:00000010:0.1:1787761547.971203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831c00.
00010000:00000010:0.1:1787761547.971205:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873d400.
00010000:00000010:0.1:1787761547.971206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876a800.
00010000:00000010:0.1:1787761547.971208:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873d200.
00010000:00000010:0.1:1787761547.971210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876b600.
00010000:00000010:0.1:1787761547.971211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873da00.
00010000:00000010:0.1:1787761547.971213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876ac00.
00010000:00000010:0.1:1787761547.971214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646f00.
00010000:00000010:0.1:1787761547.971217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3c800.
00010000:00000010:0.1:1787761547.971219:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786f300.
00010000:00000010:0.1:1787761547.971220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d7600.
00010000:00000010:0.1:1787761547.971222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d000.
00010000:00000010:0.1:1787761547.971224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7830600.
00010000:00000010:0.1:1787761547.971226:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d100.
00010000:00000010:0.1:1787761547.971227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831000.
00010000:00000010:0.1:1787761547.971229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d400.
00010000:00000010:0.1:1787761547.971230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876a200.
00010000:00000010:0.1:1787761547.971232:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466100.
00010000:00000010:0.1:1787761547.971234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876be00.
00010000:00000010:0.1:1787761547.971235:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646d00.
00010000:00000010:0.1:1787761547.971237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876a600.
00010000:00000010:0.1:1787761547.971238:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873d600.
00010000:00000010:0.1:1787761547.971240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7830400.
00010000:00000010:0.1:1787761547.971241:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646400.
00010000:00000010:0.1:1787761547.971243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3de00.
00010000:00000010:0.1:1787761547.971245:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8466a00.
00010000:00000010:0.1:1787761547.971246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876a000.
00000800:00000200:0.1:1787761551.420988:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761551.420996:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787761551.421714:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761551.421717:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787761551.421912:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761551.421914:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787761551.422096:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761551.422098:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787761556.541339:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761556.541348:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787761556.542418:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761556.542419:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787761556.542660:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761556.542662:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787761556.542871:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761556.542874:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00010000:00000010:2.1:1787761558.212211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d500.
00010000:00000010:2.1:1787761558.212224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831a00.
00010000:00000010:2.1:1787761558.212229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d800.
00010000:00000010:2.1:1787761558.212233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42000.
00010000:00000010:2.1:1787761558.212238:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d600.
00010000:00000010:2.1:1787761558.212241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7830200.
00010000:00000010:2.1:1787761558.212245:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646000.
00010000:00000010:2.1:1787761558.212250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3dc00.
00010000:00000010:2.1:1787761558.212253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646a00.
00010000:00000010:2.1:1787761558.212257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3d200.
00010000:00000010:2.1:1787761558.212260:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646300.
00010000:00000010:2.1:1787761558.212264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3c200.
00010000:00000010:2.1:1787761558.212267:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d900.
00010000:00000010:2.1:1787761558.212270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7830a00.
00010000:00000010:2.1:1787761558.212274:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4800.
00010000:00000010:2.1:1787761558.212277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d7800.
00010000:00000010:2.1:1787761558.212282:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d200.
00010000:00000010:2.1:1787761558.212286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7830c00.
00010000:00000010:2.1:1787761558.212290:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7da00.
00010000:00000010:2.1:1787761558.212293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42800.
00010000:00000010:2.1:1787761558.212297:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d786fc00.
00010000:00000010:2.1:1787761558.212300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d6200.
00010000:00000010:2.1:1787761558.212303:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646600.
00010000:00000010:2.1:1787761558.212307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3d400.
00010000:00000010:2.1:1787761558.216207:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7dc00.
00010000:00000010:2.1:1787761558.216219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42e00.
00010000:00000010:2.1:1787761558.216223:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7de00.
00010000:00000010:2.1:1787761558.216226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831400.
00010000:00000010:2.1:1787761558.216230:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7dd00.
00010000:00000010:2.1:1787761558.216233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876b000.
00010000:00000010:2.1:1787761558.216237:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7df00.
00010000:00000010:2.1:1787761558.216241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7830800.
00000800:00000200:0.1:1787761561.661248:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761561.662016:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761561.662660:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761561.662857:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761566.780937:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761566.780945:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787761566.782474:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761566.782496:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787761566.783051:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761566.783053:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787761566.783277:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761566.783279:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00010000:00000010:0.1:1787761567.427161:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1b600.
00010000:00000010:0.1:1787761567.427170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0a00.
00010000:00000010:0.1:1787761567.427173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1b700.
00010000:00000010:0.1:1787761567.427175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831800.
00010000:00000010:0.1:1787761567.427178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1b000.
00010000:00000010:0.1:1787761567.427180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb43c00.
00010000:00000010:0.1:1787761567.427183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1b100.
00010000:00000010:0.1:1787761567.427184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb43600.
00010000:00000010:0.1:1787761567.427187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2800.
00010000:00000010:0.1:1787761567.427189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8777600.
00010000:00000010:0.1:1787761567.427192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4300.
00010000:00000010:0.1:1787761567.427195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fd600.
00010000:00000010:0.1:1787761567.427197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4600.
00010000:00000010:0.1:1787761567.427199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d6000.
00010000:00000010:0.1:1787761567.427201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1b400.
00010000:00000010:0.1:1787761567.427203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7830e00.
00010000:00000010:0.1:1787761567.427205:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4700.
00010000:00000010:0.1:1787761567.427207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d6400.
00010000:00000010:0.1:1787761567.427208:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1b900.
00010000:00000010:0.1:1787761567.427210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831200.
00010000:00000010:0.1:1787761567.427211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4200.
00010000:00000010:0.1:1787761567.427213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d7200.
00010000:00000010:0.1:1787761567.427215:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4100.
00010000:00000010:0.1:1787761567.427216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d7400.
00010000:00000010:0.1:1787761567.427218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4400.
00010000:00000010:0.1:1787761567.427219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d7000.
00010000:00000010:0.1:1787761567.427221:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4900.
00010000:00000010:0.1:1787761567.427222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d6800.
00010000:00000010:0.1:1787761567.427224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4500.
00010000:00000010:0.1:1787761567.427225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7831600.
00010000:00000010:0.1:1787761567.427227:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4e00.
00010000:00000010:0.1:1787761567.427228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86d6a00.
00010000:00000010:0.1:1787761567.427230:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1b500.
00010000:00000010:0.1:1787761567.427231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7830000.
00010000:00000010:0.1:1787761567.427233:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f873df00.
00010000:00000010:0.1:1787761567.427235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f876ba00.
00010000:00000010:0.1:1787761567.427237:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d700.
00010000:00000010:0.1:1787761567.427239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb43000.
00010000:00000010:0.1:1787761567.427240:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa7d300.
00010000:00000010:0.1:1787761567.427242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb43a00.
00000800:00000200:0.1:1787761571.900829:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761571.900834:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787761571.901330:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761571.901331:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787761571.901538:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761571.901540:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787761571.901722:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761571.901724:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787761577.021344:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761577.021354:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787761577.022323:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761577.022327:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787761577.022747:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761577.022751:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787761577.023232:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761577.023236:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00010000:00000010:1.1:1787761577.668216:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518d900.
00010000:00000010:1.1:1787761577.668230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8777a00.
00010000:00000010:1.1:1787761577.668236:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646900.
00010000:00000010:1.1:1787761577.668242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c1200.
00010000:00000010:1.1:1787761577.668246:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2c00.
00010000:00000010:1.1:1787761577.668250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8777400.
00010000:00000010:1.1:1787761577.668254:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1be00.
00010000:00000010:1.1:1787761577.668259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb43e00.
00010000:00000010:1.1:1787761577.668262:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1bf00.
00010000:00000010:1.1:1787761577.668266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb43800.
00010000:00000010:1.1:1787761577.668269:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1bd00.
00010000:00000010:1.1:1787761577.668273:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c1000.
00010000:00000010:1.1:1787761577.668276:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1b200.
00010000:00000010:1.1:1787761577.668279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c1a00.
00010000:00000010:1.1:1787761577.668283:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4000.
00010000:00000010:1.1:1787761577.668286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fcc00.
00010000:00000010:1.1:1787761577.668290:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4a00.
00010000:00000010:1.1:1787761577.668293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fde00.
00010000:00000010:1.1:1787761577.668296:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1bc00.
00010000:00000010:1.1:1787761577.668299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8776400.
00010000:00000010:1.1:1787761577.671235:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646e00.
00010000:00000010:1.1:1787761577.671247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8776200.
00010000:00000010:1.1:1787761577.671253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1bb00.
00010000:00000010:1.1:1787761577.671257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42a00.
00010000:00000010:1.1:1787761577.671260:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646800.
00010000:00000010:1.1:1787761577.671264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8776c00.
00010000:00000010:1.1:1787761577.671268:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646200.
00010000:00000010:1.1:1787761577.671271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fc000.
00010000:00000010:1.1:1787761577.671274:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1b300.
00010000:00000010:1.1:1787761577.671278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb43200.
00010000:00000010:1.1:1787761577.671282:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646500.
00010000:00000010:1.1:1787761577.671285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8776000.
00000800:00000200:1.1:1787761582.141403:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761582.141415:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787761582.142565:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761582.142567:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787761582.142769:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761582.142772:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787761582.142998:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761582.143000:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787761587.261211:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761587.261217:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787761587.262080:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761587.262086:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787761587.262523:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761587.262527:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787761587.262996:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761587.262999:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00010000:00000010:0.1:1787761587.908216:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1ad00.
00010000:00000010:0.1:1787761587.908230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0400.
00010000:00000010:0.1:1787761587.908235:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2300.
00010000:00000010:0.1:1787761587.908240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a9400.
00010000:00000010:0.1:1787761587.908244:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2a00.
00010000:00000010:0.1:1787761587.908248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a8800.
00010000:00000010:0.1:1787761587.908251:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1af00.
00010000:00000010:0.1:1787761587.908254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0c00.
00010000:00000010:0.1:1787761587.908258:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1ac00.
00010000:00000010:0.1:1787761587.908268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a9e00.
00010000:00000010:0.1:1787761587.908272:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1a800.
00010000:00000010:0.1:1787761587.908277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a8a00.
00010000:00000010:0.1:1787761587.908280:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4d00.
00010000:00000010:0.1:1787761587.908284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fd800.
00010000:00000010:0.1:1787761587.908287:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1a600.
00010000:00000010:0.1:1787761587.908291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c1800.
00010000:00000010:0.1:1787761587.912192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1a700.
00010000:00000010:0.1:1787761587.912203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0e00.
00010000:00000010:0.1:1787761587.912208:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1a200.
00010000:00000010:0.1:1787761587.912212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0800.
00010000:00000010:0.1:1787761587.912215:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1a100.
00010000:00000010:0.1:1787761587.912219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a9a00.
00010000:00000010:0.1:1787761587.912223:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2000.
00010000:00000010:0.1:1787761587.912227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42c00.
00010000:00000010:0.1:1787761587.912230:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2100.
00010000:00000010:0.1:1787761587.912234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42400.
00010000:00000010:0.1:1787761587.912237:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2900.
00010000:00000010:0.1:1787761587.912240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c1600.
00010000:00000010:0.1:1787761587.912244:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1a400.
00010000:00000010:0.1:1787761587.912247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c1e00.
00010000:00000010:0.1:1787761587.912250:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f4646700.
00010000:00000010:0.1:1787761587.912254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c1c00.
00010000:00000010:0.1:1787761587.912257:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2500.
00010000:00000010:0.1:1787761587.912260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42600.
00010000:00000010:0.1:1787761587.912263:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5e800.
00010000:00000010:0.1:1787761587.912267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb42200.
00010000:00000010:0.1:1787761587.912270:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518df00.
00010000:00000010:0.1:1787761587.912275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8777000.
00010000:00000010:0.1:1787761587.912279:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1b800.
00010000:00000010:0.1:1787761587.912282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0000.
00010000:00000010:0.1:1787761587.912285:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1ba00.
00010000:00000010:0.1:1787761587.912289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb43400.
00000800:00000200:1.1:1787761592.380841:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761592.380848:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787761592.381394:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761592.381398:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787761592.381635:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761592.381637:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787761592.381843:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761592.381845:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787761597.500757:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761597.500762:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787761597.501170:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761597.501172:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787761597.501388:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761597.501391:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787761597.501571:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761597.501574:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00010000:00000010:2.1:1787761598.147162:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c400.
00010000:00000010:2.1:1787761598.147170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fda00.
00010000:00000010:2.1:1787761598.147173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870cd00.
00010000:00000010:2.1:1787761598.147175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fc600.
00010000:00000010:2.1:1787761598.147178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98700.
00010000:00000010:2.1:1787761598.147180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1a000.
00010000:00000010:2.1:1787761598.147183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98200.
00010000:00000010:2.1:1787761598.147184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1a600.
00010000:00000010:2.1:1787761598.147186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98100.
00010000:00000010:2.1:1787761598.147188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8777c00.
00010000:00000010:2.1:1787761598.147191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98900.
00010000:00000010:2.1:1787761598.147193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a8c00.
00010000:00000010:2.1:1787761598.147196:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1ae00.
00010000:00000010:2.1:1787761598.147198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1ae00.
00010000:00000010:2.1:1787761598.147199:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1a000.
00010000:00000010:2.1:1787761598.147201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1b400.
00010000:00000010:2.1:1787761598.147204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1a900.
00010000:00000010:2.1:1787761598.147206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb3c000.
00010000:00000010:2.1:1787761598.151162:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1aa00.
00010000:00000010:2.1:1787761598.151168:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a8000.
00010000:00000010:2.1:1787761598.151171:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2200.
00010000:00000010:2.1:1787761598.151173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a9600.
00010000:00000010:2.1:1787761598.151175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1a300.
00010000:00000010:2.1:1787761598.151177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0200.
00010000:00000010:2.1:1787761598.151179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5e500.
00010000:00000010:2.1:1787761598.151181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8777800.
00010000:00000010:2.1:1787761598.151183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1ab00.
00010000:00000010:2.1:1787761598.151185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a9800.
00010000:00000010:2.1:1787761598.151187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5ee00.
00010000:00000010:2.1:1787761598.151189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8776800.
00010000:00000010:2.1:1787761598.151191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4c00.
00010000:00000010:2.1:1787761598.151193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fd000.
00010000:00000010:2.1:1787761598.151194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4f00.
00010000:00000010:2.1:1787761598.151196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8777200.
00010000:00000010:2.1:1787761598.151197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5eb00.
00010000:00000010:2.1:1787761598.151199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8776600.
00010000:00000010:2.1:1787761598.151200:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfad4b00.
00010000:00000010:2.1:1787761598.151202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fce00.
00010000:00000010:2.1:1787761598.151204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa1a500.
00010000:00000010:2.1:1787761598.151206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c1400.
00010000:00000010:2.1:1787761598.151208:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2700.
00010000:00000010:2.1:1787761598.151210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a8200.
00010000:00000010:2.1:1787761598.151212:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2600.
00010000:00000010:2.1:1787761598.151214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f85c0600.
00000800:00000200:1.1:1787761602.622007:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761602.622011:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787761602.622534:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761602.622538:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787761602.623017:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761602.623020:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787761602.623464:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761602.623467:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00010000:00000010:2.1:1787761607.364210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c600.
00010000:00000010:2.1:1787761607.364222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fd200.
00010000:00000010:2.1:1787761607.364228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5e700.
00010000:00000010:2.1:1787761607.364233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877f400.
00010000:00000010:2.1:1787761607.364239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98a00.
00010000:00000010:2.1:1787761607.364243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a9000.
00010000:00000010:2.1:1787761607.364247:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5e300.
00010000:00000010:2.1:1787761607.364251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877f800.
00010000:00000010:2.1:1787761607.364254:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98c00.
00010000:00000010:2.1:1787761607.364258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877e200.
00010000:00000010:2.1:1787761607.364261:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98f00.
00010000:00000010:2.1:1787761607.364264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1a200.
00010000:00000010:2.1:1787761607.364268:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2e00.
00010000:00000010:2.1:1787761607.364271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1a400.
00010000:00000010:2.1:1787761607.364274:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5ea00.
00010000:00000010:2.1:1787761607.364278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877ec00.
00010000:00000010:2.1:1787761607.364281:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5e000.
00010000:00000010:2.1:1787761607.364284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877fc00.
00010000:00000010:2.1:1787761607.364287:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5e100.
00010000:00000010:2.1:1787761607.364290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8776e00.
00010000:00000010:2.1:1787761607.364294:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2f00.
00010000:00000010:2.1:1787761607.364297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a9200.
00010000:00000010:2.1:1787761607.364300:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98500.
00010000:00000010:2.1:1787761607.364303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1b800.
00010000:00000010:2.1:1787761607.368208:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5e400.
00010000:00000010:2.1:1787761607.368219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7923000.
00010000:00000010:2.1:1787761607.368223:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5e900.
00010000:00000010:2.1:1787761607.368227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8777e00.
00010000:00000010:2.1:1787761607.368231:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98b00.
00010000:00000010:2.1:1787761607.368234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a8e00.
00010000:00000010:2.1:1787761607.368238:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c000.
00010000:00000010:2.1:1787761607.368241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fca00.
00010000:00000010:2.1:1787761607.368244:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2d00.
00010000:00000010:2.1:1787761607.368248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a8400.
00010000:00000010:2.1:1787761607.368251:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98800.
00010000:00000010:2.1:1787761607.368254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8776a00.
00010000:00000010:2.1:1787761607.368258:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c100.
00010000:00000010:2.1:1787761607.368262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fc400.
00000800:00000200:1.1:1787761607.741292:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761607.741299:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787761607.742356:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761607.742360:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787761607.742743:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761607.742746:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787761607.743219:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761607.743223:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787761612.861170:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761612.861182:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787761612.862447:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761612.862450:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787761612.862746:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761612.862748:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787761612.863007:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761612.863009:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00010000:00000010:0.1:1787761617.603171:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310300.
00010000:00000010:0.1:1787761617.603180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b9c00.
00010000:00000010:0.1:1787761617.603184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646000.
00010000:00000010:0.1:1787761617.603186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877f200.
00010000:00000010:0.1:1787761617.603190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870ce00.
00010000:00000010:0.1:1787761617.603192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877e800.
00010000:00000010:0.1:1787761617.607164:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c500.
00010000:00000010:0.1:1787761617.607170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fc800.
00010000:00000010:0.1:1787761617.607173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646100.
00010000:00000010:0.1:1787761617.607175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1aa00.
00010000:00000010:0.1:1787761617.607178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646900.
00010000:00000010:0.1:1787761617.607180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b9a00.
00010000:00000010:0.1:1787761617.607182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c200.
00010000:00000010:0.1:1787761617.607184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fdc00.
00010000:00000010:0.1:1787761617.607186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870ca00.
00010000:00000010:0.1:1787761617.607188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fc200.
00010000:00000010:0.1:1787761617.607190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646d00.
00010000:00000010:0.1:1787761617.607192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b8400.
00010000:00000010:0.1:1787761617.607194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310600.
00010000:00000010:0.1:1787761617.607195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1bc00.
00010000:00000010:0.1:1787761617.607197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5ed00.
00010000:00000010:0.1:1787761617.607199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b9200.
00010000:00000010:0.1:1787761617.607201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870cb00.
00010000:00000010:0.1:1787761617.607203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877e600.
00010000:00000010:0.1:1787761617.607205:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646e00.
00010000:00000010:0.1:1787761617.607207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f49fd400.
00010000:00000010:0.1:1787761617.607210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2400.
00010000:00000010:0.1:1787761617.607212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877f000.
00010000:00000010:0.1:1787761617.607213:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5e200.
00010000:00000010:0.1:1787761617.607215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877fa00.
00010000:00000010:0.1:1787761617.607216:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646b00.
00010000:00000010:0.1:1787761617.607218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b9000.
00010000:00000010:0.1:1787761617.607220:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98600.
00010000:00000010:0.1:1787761617.607221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b8e00.
00010000:00000010:0.1:1787761617.607224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98e00.
00010000:00000010:0.1:1787761617.607226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b8a00.
00010000:00000010:0.1:1787761617.607229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98300.
00010000:00000010:0.1:1787761617.607232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b8000.
00010000:00000010:0.1:1787761617.607234:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98000.
00010000:00000010:0.1:1787761617.607236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1b600.
00010000:00000010:0.1:1787761617.607238:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e92c2b00.
00010000:00000010:0.1:1787761617.607241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a9c00.
00010000:00000010:0.1:1787761617.607244:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98d00.
00010000:00000010:0.1:1787761617.607247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1b200.
00010000:00000010:0.1:1787761617.607250:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa98400.
00010000:00000010:0.1:1787761617.607253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87a8600.
00000800:00000200:0.1:1787761617.981344:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761617.981356:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787761617.982409:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761617.982413:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787761617.982833:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761617.982836:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787761617.983274:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761617.983277:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787761623.100836:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761623.100843:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787761623.101388:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761623.101391:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787761623.101727:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761623.101728:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787761623.101937:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761623.101940:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00010000:00000010:2.1:1787761627.844164:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4700.
00010000:00000010:2.1:1787761627.844172:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f3800.
00010000:00000010:2.1:1787761627.844176:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646500.
00010000:00000010:2.1:1787761627.844179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1be00.
00010000:00000010:2.1:1787761627.844182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646c00.
00010000:00000010:2.1:1787761627.844184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1ac00.
00010000:00000010:2.1:1787761627.844186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c300.
00010000:00000010:2.1:1787761627.844189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cfe00.
00010000:00000010:2.1:1787761627.844191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646f00.
00010000:00000010:2.1:1787761627.844193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877fe00.
00010000:00000010:2.1:1787761627.844196:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870cc00.
00010000:00000010:2.1:1787761627.844198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cfc00.
00010000:00000010:2.1:1787761627.844200:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646400.
00010000:00000010:2.1:1787761627.844202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1ba00.
00010000:00000010:2.1:1787761627.844204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310f00.
00010000:00000010:2.1:1787761627.844206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b8800.
00010000:00000010:2.1:1787761627.844208:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646200.
00010000:00000010:2.1:1787761627.844211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877ea00.
00010000:00000010:2.1:1787761627.844212:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646700.
00010000:00000010:2.1:1787761627.844214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1a800.
00010000:00000010:2.1:1787761627.844216:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646800.
00010000:00000010:2.1:1787761627.844218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b9800.
00010000:00000010:2.1:1787761627.844220:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646300.
00010000:00000010:2.1:1787761627.844221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b8200.
00010000:00000010:2.1:1787761627.844223:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4200.
00010000:00000010:2.1:1787761627.844226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877e400.
00010000:00000010:2.1:1787761627.844228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646600.
00010000:00000010:2.1:1787761627.844230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877ee00.
00010000:00000010:2.1:1787761627.844232:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310c00.
00010000:00000010:2.1:1787761627.844234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b9400.
00010000:00000010:2.1:1787761627.844238:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5e600.
00010000:00000010:2.1:1787761627.844240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877e000.
00010000:00000010:2.1:1787761627.844242:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310800.
00010000:00000010:2.1:1787761627.844244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b9600.
00010000:00000010:2.1:1787761627.844247:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef646a00.
00010000:00000010:2.1:1787761627.844249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b9e00.
00010000:00000010:2.1:1787761627.844250:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5ec00.
00010000:00000010:2.1:1787761627.844252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b8c00.
00010000:00000010:2.1:1787761627.844254:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa5ef00.
00010000:00000010:2.1:1787761627.844256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f877f600.
00000800:00000200:1.1:1787761628.221305:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761628.221923:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761628.222291:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761628.222627:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761633.340862:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761633.340869:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787761633.341527:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761633.341544:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787761633.341848:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761633.341851:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787761633.342081:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761633.342083:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00010000:00000010:2.1:1787761638.084186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce100.
00010000:00000010:2.1:1787761638.084197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8785400.
00010000:00000010:2.1:1787761638.084202:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310200.
00010000:00000010:2.1:1787761638.084206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865c000.
00010000:00000010:2.1:1787761638.084210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4b00.
00010000:00000010:2.1:1787761638.084214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865c200.
00010000:00000010:2.1:1787761638.084217:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce400.
00010000:00000010:2.1:1787761638.084221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8784400.
00010000:00000010:2.1:1787761638.084225:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c800.
00010000:00000010:2.1:1787761638.084229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865c600.
00010000:00000010:2.1:1787761638.088183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4800.
00010000:00000010:2.1:1787761638.088195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f3400.
00010000:00000010:2.1:1787761638.088202:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ced00.
00010000:00000010:2.1:1787761638.088206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8784200.
00010000:00000010:2.1:1787761638.088210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011cef00.
00010000:00000010:2.1:1787761638.088213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8784c00.
00010000:00000010:2.1:1787761638.088217:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011cec00.
00010000:00000010:2.1:1787761638.088220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865c400.
00010000:00000010:2.1:1787761638.088224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310100.
00010000:00000010:2.1:1787761638.088228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865d400.
00010000:00000010:2.1:1787761638.088231:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870cf00.
00010000:00000010:2.1:1787761638.088235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87ce400.
00010000:00000010:2.1:1787761638.088239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ceb00.
00010000:00000010:2.1:1787761638.088243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865d600.
00010000:00000010:2.1:1787761638.088246:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce300.
00010000:00000010:2.1:1787761638.088249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865c800.
00010000:00000010:2.1:1787761638.088253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011cea00.
00010000:00000010:2.1:1787761638.088256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfb1b000.
00010000:00000010:2.1:1787761638.088260:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c900.
00010000:00000010:2.1:1787761638.088263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cea00.
00010000:00000010:2.1:1787761638.088266:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce000.
00010000:00000010:2.1:1787761638.088269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865cc00.
00010000:00000010:2.1:1787761638.088272:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310900.
00010000:00000010:2.1:1787761638.088275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865dc00.
00010000:00000010:2.1:1787761638.088278:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4000.
00010000:00000010:2.1:1787761638.088281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865d800.
00000800:00000200:1.1:1787761638.460844:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761638.460851:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787761638.461288:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761638.461290:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787761638.461491:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761638.461492:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787761638.461758:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761638.461760:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787761643.581369:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761643.581380:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787761643.582456:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761643.582485:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787761643.582861:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761643.582864:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787761643.583239:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761643.583242:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00010000:00000010:0.1:1787761647.301185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310a00.
00010000:00000010:0.1:1787761647.301196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53aae00.
00010000:00000010:0.1:1787761647.301204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b900.
00010000:00000010:0.1:1787761647.301209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cf400.
00010000:00000010:0.1:1787761647.301213:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870bd00.
00010000:00000010:0.1:1787761647.301218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cee00.
00010000:00000010:0.1:1787761647.301225:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce500.
00010000:00000010:0.1:1787761647.301234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f3600.
00010000:00000010:0.1:1787761647.301243:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce600.
00010000:00000010:0.1:1787761647.301254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8785200.
00010000:00000010:0.1:1787761647.301262:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310d00.
00010000:00000010:0.1:1787761647.301269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751e78b8600.
00010000:00000010:0.1:1787761647.301273:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce200.
00010000:00000010:0.1:1787761647.301276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f3200.
00010000:00000010:0.1:1787761647.301280:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310500.
00010000:00000010:0.1:1787761647.301284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865ce00.
00010000:00000010:0.1:1787761647.301287:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce800.
00010000:00000010:0.1:1787761647.301290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8785600.
00010000:00000010:0.1:1787761647.301293:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870be00.
00010000:00000010:0.1:1787761647.301296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865d200.
00010000:00000010:0.1:1787761647.301300:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce700.
00010000:00000010:0.1:1787761647.301303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2800.
00010000:00000010:0.1:1787761647.301306:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310400.
00010000:00000010:0.1:1787761647.301309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865ca00.
00010000:00000010:0.1:1787761647.301312:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4f00.
00010000:00000010:0.1:1787761647.301316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865d000.
00010000:00000010:0.1:1787761647.301319:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870bc00.
00010000:00000010:0.1:1787761647.301322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87ce800.
00010000:00000010:0.1:1787761647.301326:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310000.
00010000:00000010:0.1:1787761647.301330:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865da00.
00010000:00000010:0.1:1787761647.301333:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011cee00.
00010000:00000010:0.1:1787761647.301336:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8784800.
00010000:00000010:0.1:1787761647.301340:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9752011ce900.
00010000:00000010:0.1:1787761647.301343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8785a00.
00010000:00000010:0.1:1787761647.301347:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870c700.
00010000:00000010:0.1:1787761647.301350:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cf000.
00010000:00000010:0.1:1787761647.301353:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4e00.
00010000:00000010:0.1:1787761647.301356:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f865de00.
00000800:00000200:1.1:1787761648.701018:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761648.701025:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787761648.701567:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761648.701569:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787761648.701837:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761648.701839:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787761648.702049:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761648.702051:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787761653.820936:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761653.820943:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787761653.821409:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761653.821411:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787761653.821649:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761653.821651:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787761653.821847:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761653.821849:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00010000:00000010:3.1:1787761657.539172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b600.
00010000:00000010:3.1:1787761657.539180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87ce600.
00010000:00000010:3.1:1787761657.539183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4a00.
00010000:00000010:3.1:1787761657.539186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f3a00.
00010000:00000010:3.1:1787761657.539189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4100.
00010000:00000010:3.1:1787761657.539191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f3000.
00010000:00000010:3.1:1787761657.539193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85bd00.
00010000:00000010:3.1:1787761657.539196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8785e00.
00010000:00000010:3.1:1787761657.539198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4300.
00010000:00000010:3.1:1787761657.539200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2000.
00010000:00000010:3.1:1787761657.539202:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4c00.
00010000:00000010:3.1:1787761657.539203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2200.
00010000:00000010:3.1:1787761657.539205:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85bf00.
00010000:00000010:3.1:1787761657.539207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53aa800.
00010000:00000010:3.1:1787761657.539209:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310b00.
00010000:00000010:3.1:1787761657.539211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53aa600.
00010000:00000010:3.1:1787761657.539213:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b700.
00010000:00000010:3.1:1787761657.539215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87ce000.
00010000:00000010:3.1:1787761657.539217:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85be00.
00010000:00000010:3.1:1787761657.539219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2a00.
00010000:00000010:3.1:1787761657.539222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310e00.
00010000:00000010:3.1:1787761657.539225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53ab000.
00010000:00000010:3.1:1787761657.539228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751ef310700.
00010000:00000010:3.1:1787761657.539230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53aa200.
00010000:00000010:3.1:1787761657.539233:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4d00.
00010000:00000010:3.1:1787761657.539236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f3e00.
00010000:00000010:3.1:1787761657.539239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b200.
00010000:00000010:3.1:1787761657.539270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53ab200.
00010000:00000010:3.1:1787761657.539272:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4400.
00010000:00000010:3.1:1787761657.539274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f3c00.
00010000:00000010:3.1:1787761657.539276:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b800.
00010000:00000010:3.1:1787761657.539278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53aba00.
00010000:00000010:3.1:1787761657.543173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b400.
00010000:00000010:3.1:1787761657.543180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cf800.
00010000:00000010:3.1:1787761657.543183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b600.
00010000:00000010:3.1:1787761657.543186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8784600.
00010000:00000010:3.1:1787761657.543189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4900.
00010000:00000010:3.1:1787761657.543192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53aa400.
00000800:00000200:3.1:1787761658.941163:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787761658.941167:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787761658.941582:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787761658.941584:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787761658.941785:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787761658.941787:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787761658.942015:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787761658.942017:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787761664.061415:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787761664.061422:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1787761664.062376:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787761664.062379:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1787761664.062756:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787761664.062759:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1787761664.063189:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787761664.063193:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00010000:00000010:0.1:1787761667.780222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b000.
00010000:00000010:0.1:1787761667.780234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53ab800.
00010000:00000010:0.1:1787761667.780240:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870bb00.
00010000:00000010:0.1:1787761667.780244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cf200.
00010000:00000010:0.1:1787761667.780248:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b100.
00010000:00000010:0.1:1787761667.780251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53ab400.
00010000:00000010:0.1:1787761667.780255:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85ba00.
00010000:00000010:0.1:1787761667.780258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8784a00.
00010000:00000010:0.1:1787761667.780262:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b300.
00010000:00000010:0.1:1787761667.780265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8785000.
00010000:00000010:0.1:1787761667.784223:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85bb00.
00010000:00000010:0.1:1787761667.784234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53abe00.
00010000:00000010:0.1:1787761667.784239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85bc00.
00010000:00000010:0.1:1787761667.784243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53aa000.
00010000:00000010:0.1:1787761667.784247:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2800.
00010000:00000010:0.1:1787761667.784341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791a00.
00010000:00000010:0.1:1787761667.784348:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b100.
00010000:00000010:0.1:1787761667.784352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8784000.
00010000:00000010:0.1:1787761667.784355:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2300.
00010000:00000010:0.1:1787761667.784359:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8790e00.
00010000:00000010:0.1:1787761667.784362:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b000.
00010000:00000010:0.1:1787761667.784366:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cfa00.
00010000:00000010:0.1:1787761667.784369:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b700.
00010000:00000010:0.1:1787761667.784374:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8784e00.
00010000:00000010:0.1:1787761667.784377:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2600.
00010000:00000010:0.1:1787761667.784381:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53abc00.
00010000:00000010:0.1:1787761667.784384:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2700.
00010000:00000010:0.1:1787761667.784388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2400.
00010000:00000010:0.1:1787761667.784392:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870ba00.
00010000:00000010:0.1:1787761667.784395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87ce200.
00010000:00000010:0.1:1787761667.784398:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b800.
00010000:00000010:0.1:1787761667.784401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53ab600.
00010000:00000010:0.1:1787761667.784405:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4600.
00010000:00000010:0.1:1787761667.784408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2600.
00010000:00000010:0.1:1787761667.784411:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f85b4500.
00010000:00000010:0.1:1787761667.784414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2e00.
00010000:00000010:0.1:1787761667.784417:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b200.
00010000:00000010:0.1:1787761667.784421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8785800.
00010000:00000010:0.1:1787761667.784424:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b400.
00010000:00000010:0.1:1787761667.784427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8785c00.
00010000:00000010:0.1:1787761667.784431:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b900.
00010000:00000010:0.1:1787761667.784434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53aaa00.
00000800:00000200:1.1:1787761669.181677:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761669.181687:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:1.1:1787761669.182808:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761669.182812:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:1.1:1787761669.183415:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761669.183418:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:1.1:1787761669.183791:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761669.183794:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:1.1:1787761674.301460:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761674.301465:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787761674.302123:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761674.302129:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787761674.302694:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761674.302697:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787761674.303221:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761674.303225:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00010000:00000010:3.1:1787761678.019170:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53800.
00010000:00000010:3.1:1787761678.019179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbfe200.
00010000:00000010:3.1:1787761678.019182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53600.
00010000:00000010:3.1:1787761678.019185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d6800.
00010000:00000010:3.1:1787761678.019188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772600.
00010000:00000010:3.1:1787761678.019190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbff000.
00010000:00000010:3.1:1787761678.019192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772a00.
00010000:00000010:3.1:1787761678.019194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbff400.
00010000:00000010:3.1:1787761678.019195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772700.
00010000:00000010:3.1:1787761678.019197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbfe400.
00010000:00000010:3.1:1787761678.019199:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772000.
00010000:00000010:3.1:1787761678.019201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbffe00.
00010000:00000010:3.1:1787761678.019202:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53700.
00010000:00000010:3.1:1787761678.019204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d7a00.
00010000:00000010:3.1:1787761678.019206:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772200.
00010000:00000010:3.1:1787761678.019207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d6e00.
00010000:00000010:3.1:1787761678.019209:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53200.
00010000:00000010:3.1:1787761678.019211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d7400.
00010000:00000010:3.1:1787761678.019212:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2e00.
00010000:00000010:3.1:1787761678.019214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8790000.
00010000:00000010:3.1:1787761678.019216:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772100.
00010000:00000010:3.1:1787761678.019228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791200.
00010000:00000010:3.1:1787761678.019232:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870bf00.
00010000:00000010:3.1:1787761678.019235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cec00.
00010000:00000010:3.1:1787761678.019237:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53100.
00010000:00000010:3.1:1787761678.019238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d6400.
00010000:00000010:3.1:1787761678.019240:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53900.
00010000:00000010:3.1:1787761678.019241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8790600.
00010000:00000010:3.1:1787761678.019243:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b500.
00010000:00000010:3.1:1787761678.019245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87cf600.
00010000:00000010:3.1:1787761678.019246:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53500.
00010000:00000010:3.1:1787761678.019266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d6200.
00010000:00000010:3.1:1787761678.023168:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2c00.
00010000:00000010:3.1:1787761678.023174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8790800.
00010000:00000010:3.1:1787761678.023176:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2b00.
00010000:00000010:3.1:1787761678.023178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d6a00.
00010000:00000010:3.1:1787761678.023180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53f00.
00010000:00000010:3.1:1787761678.023183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c53aac00.
00010000:00000010:3.1:1787761678.023185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53e00.
00010000:00000010:3.1:1787761678.023187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d7e00.
00010000:00000010:3.1:1787761678.023189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cf85b500.
00010000:00000010:3.1:1787761678.023190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d7c00.
00000800:00000200:1.1:1787761679.421292:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761679.421308:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787761679.422253:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761679.422256:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787761679.422632:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761679.422636:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787761679.422976:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761679.422980:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787761684.541442:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761684.541449:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787761684.542308:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761684.542312:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787761684.542674:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761684.542677:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787761684.543081:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761684.543084:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00010000:00000010:3.1:1787761688.259186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772900.
00010000:00000010:3.1:1787761688.259193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbff600.
00010000:00000010:3.1:1787761688.259196:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772d00.
00010000:00000010:3.1:1787761688.259198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8790400.
00010000:00000010:3.1:1787761688.259200:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772500.
00010000:00000010:3.1:1787761688.259202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d6c00.
00010000:00000010:3.1:1787761688.259205:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797a900.
00010000:00000010:3.1:1787761688.259207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d7000.
00010000:00000010:3.1:1787761688.259209:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2100.
00010000:00000010:3.1:1787761688.259211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84acc00.
00010000:00000010:3.1:1787761688.259213:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53c00.
00010000:00000010:3.1:1787761688.259216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbffc00.
00010000:00000010:3.1:1787761688.259218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2900.
00010000:00000010:3.1:1787761688.259220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8790200.
00010000:00000010:3.1:1787761688.259222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53a00.
00010000:00000010:3.1:1787761688.259224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbff200.
00010000:00000010:3.1:1787761688.259225:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2500.
00010000:00000010:3.1:1787761688.259227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8790a00.
00010000:00000010:3.1:1787761688.259228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53400.
00010000:00000010:3.1:1787761688.259230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ade00.
00010000:00000010:3.1:1787761688.259232:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53d00.
00010000:00000010:3.1:1787761688.259246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d7600.
00010000:00000010:3.1:1787761688.259248:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128bb00.
00010000:00000010:3.1:1787761688.259250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d7200.
00010000:00000010:3.1:1787761688.259252:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870b300.
00010000:00000010:3.1:1787761688.259262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ad600.
00010000:00000010:3.1:1787761688.259265:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53000.
00010000:00000010:3.1:1787761688.259266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d6000.
00010000:00000010:3.1:1787761688.259268:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53300.
00010000:00000010:3.1:1787761688.259270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d7800.
00010000:00000010:3.1:1787761688.259272:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772e00.
00010000:00000010:3.1:1787761688.259274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbfec00.
00010000:00000010:3.1:1787761688.259275:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772c00.
00010000:00000010:3.1:1787761688.259277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ad200.
00010000:00000010:3.1:1787761688.263172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772800.
00010000:00000010:3.1:1787761688.263177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbfe600.
00010000:00000010:3.1:1787761688.263180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cfe53b00.
00010000:00000010:3.1:1787761688.263181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87d6600.
00000800:00000200:1.1:1787761689.661218:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761689.661233:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787761689.662654:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761689.662656:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787761689.662880:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761689.662882:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787761689.663168:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761689.663170:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787761694.781224:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761694.783419:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761694.783620:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761694.783746:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00010000:00000010:3.1:1787761697.475167:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797af00.
00010000:00000010:3.1:1787761697.475175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ada00.
00010000:00000010:3.1:1787761697.475178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797ae00.
00010000:00000010:3.1:1787761697.475181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8790c00.
00010000:00000010:3.1:1787761697.475183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2a00.
00010000:00000010:3.1:1787761697.475186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ace00.
00010000:00000010:3.1:1787761697.475188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2d00.
00010000:00000010:3.1:1787761697.475190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbfe000.
00010000:00000010:3.1:1787761697.475192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b400.
00010000:00000010:3.1:1787761697.475195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b0400.
00010000:00000010:3.1:1787761697.475197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797a800.
00010000:00000010:3.1:1787761697.475200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbfe800.
00010000:00000010:3.1:1787761697.475202:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b900.
00010000:00000010:3.1:1787761697.475204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ac400.
00010000:00000010:3.1:1787761697.475205:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128bd00.
00010000:00000010:3.1:1787761697.475224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ad800.
00010000:00000010:3.1:1787761697.475226:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772f00.
00010000:00000010:3.1:1787761697.475228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbff800.
00010000:00000010:3.1:1787761697.475230:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b500.
00010000:00000010:3.1:1787761697.475232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ac200.
00010000:00000010:3.1:1787761697.475234:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772400.
00010000:00000010:3.1:1787761697.475236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbfea00.
00010000:00000010:3.1:1787761697.475237:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797a600.
00010000:00000010:3.1:1787761697.475239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791e00.
00010000:00000010:3.1:1787761697.475241:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772300.
00010000:00000010:3.1:1787761697.475242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbfee00.
00010000:00000010:3.1:1787761697.475244:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797a700.
00010000:00000010:3.1:1787761697.475246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b1c00.
00010000:00000010:3.1:1787761697.475248:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8772b00.
00010000:00000010:3.1:1787761697.475249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751dfbffa00.
00010000:00000010:3.1:1787761697.475251:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797a200.
00010000:00000010:3.1:1787761697.475253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b0000.
00000800:00000200:1.1:1787761699.901369:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761699.901377:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787761699.902929:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761699.902933:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787761699.903402:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761699.903406:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787761699.903792:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761699.903795:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787761705.021413:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761705.021432:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787761705.022242:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761705.022245:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787761705.022664:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761705.022667:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787761705.023010:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761705.023013:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00010000:00000010:0.1:1787761707.716188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128bf00.
00010000:00000010:0.1:1787761707.716199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7835c00.
00010000:00000010:0.1:1787761707.716208:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2000.
00010000:00000010:0.1:1787761707.716214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873b400.
00010000:00000010:0.1:1787761707.716221:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b200.
00010000:00000010:0.1:1787761707.716226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7834c00.
00010000:00000010:0.1:1787761707.716232:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797a100.
00010000:00000010:0.1:1787761707.716238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7834a00.
00010000:00000010:0.1:1787761707.716243:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797a000.
00010000:00000010:0.1:1787761707.716249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7835400.
00010000:00000010:0.1:1787761707.716255:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2f00.
00010000:00000010:0.1:1787761707.716261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873b800.
00010000:00000010:0.1:1787761707.716267:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2200.
00010000:00000010:0.1:1787761707.716274:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791400.
00010000:00000010:0.1:1787761707.716279:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2a00.
00010000:00000010:0.1:1787761707.720182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7834000.
00010000:00000010:0.1:1787761707.720191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b700.
00010000:00000010:0.1:1787761707.720196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ac000.
00010000:00000010:0.1:1787761707.720201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b300.
00010000:00000010:0.1:1787761707.720207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791c00.
00010000:00000010:0.1:1787761707.720213:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797ab00.
00010000:00000010:0.1:1787761707.720219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791000.
00010000:00000010:0.1:1787761707.720224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2600.
00010000:00000010:0.1:1787761707.720230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7835600.
00010000:00000010:0.1:1787761707.720235:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128ba00.
00010000:00000010:0.1:1787761707.720240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ad400.
00010000:00000010:0.1:1787761707.720245:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b000.
00010000:00000010:0.1:1787761707.720250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ac800.
00010000:00000010:0.1:1787761707.720255:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797ac00.
00010000:00000010:0.1:1787761707.720261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791600.
00010000:00000010:0.1:1787761707.720266:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797a500.
00010000:00000010:0.1:1787761707.720272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b0e00.
00000800:00000200:1.1:1787761710.141014:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761710.141021:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787761710.141784:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761710.141789:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787761710.142106:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761710.142107:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787761710.142402:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761710.142404:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787761715.260697:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761715.260703:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787761715.261214:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761715.261216:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787761715.261406:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761715.261408:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787761715.261581:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761715.261583:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00010000:00000010:3.1:1787761717.955167:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03800.
00010000:00000010:3.1:1787761717.955174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873b600.
00010000:00000010:3.1:1787761717.955177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128be00.
00010000:00000010:3.1:1787761717.955179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ad000.
00010000:00000010:3.1:1787761717.955182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945a00.
00010000:00000010:3.1:1787761717.955184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b0c00.
00010000:00000010:3.1:1787761717.955186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03300.
00010000:00000010:3.1:1787761717.955188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7835e00.
00010000:00000010:3.1:1787761717.955190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128bc00.
00010000:00000010:3.1:1787761717.955192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84aca00.
00010000:00000010:3.1:1787761717.955194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945200.
00010000:00000010:3.1:1787761717.955195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7835800.
00010000:00000010:3.1:1787761717.955197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945400.
00010000:00000010:3.1:1787761717.955199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b1400.
00010000:00000010:3.1:1787761717.955200:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2b00.
00010000:00000010:3.1:1787761717.955213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873a600.
00010000:00000010:3.1:1787761717.955215:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe2400.
00010000:00000010:3.1:1787761717.955217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873b000.
00010000:00000010:3.1:1787761717.955219:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2300.
00010000:00000010:3.1:1787761717.955221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7835000.
00010000:00000010:3.1:1787761717.955222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945900.
00010000:00000010:3.1:1787761717.955224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7835200.
00010000:00000010:3.1:1787761717.955226:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797a300.
00010000:00000010:3.1:1787761717.955228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7834600.
00010000:00000010:3.1:1787761717.955229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b800.
00010000:00000010:3.1:1787761717.955231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b1600.
00010000:00000010:3.1:1787761717.955233:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797aa00.
00010000:00000010:3.1:1787761717.955234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b1200.
00010000:00000010:3.1:1787761717.955236:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797a400.
00010000:00000010:3.1:1787761717.955237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b0600.
00010000:00000010:3.1:1787761717.955239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d797ad00.
00010000:00000010:3.1:1787761717.955241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873ba00.
00000800:00000200:1.1:1787761720.380856:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761720.380861:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787761720.381358:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761720.381360:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787761720.381612:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761720.381614:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787761720.381835:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761720.381837:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787761725.501492:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761725.501513:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787761725.502548:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761725.502552:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787761725.502949:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761725.502953:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787761725.503244:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761725.503247:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00010000:00000010:1.1:1787761728.195163:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945100.
00010000:00000010:1.1:1787761728.195171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b0200.
00010000:00000010:1.1:1787761728.195174:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945700.
00010000:00000010:1.1:1787761728.195177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b1a00.
00010000:00000010:1.1:1787761728.195179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2d00.
00010000:00000010:1.1:1787761728.195182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873aa00.
00010000:00000010:1.1:1787761728.195185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2500.
00010000:00000010:1.1:1787761728.195187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7835a00.
00010000:00000010:1.1:1787761728.195190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945b00.
00010000:00000010:1.1:1787761728.195192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b1e00.
00010000:00000010:1.1:1787761728.195194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03e00.
00010000:00000010:1.1:1787761728.195196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873ac00.
00010000:00000010:1.1:1787761728.195197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945500.
00010000:00000010:1.1:1787761728.195199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873bc00.
00010000:00000010:1.1:1787761728.195201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae100.
00010000:00000010:1.1:1787761728.195203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84adc00.
00010000:00000010:1.1:1787761728.199160:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2f00.
00010000:00000010:1.1:1787761728.199166:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873be00.
00010000:00000010:1.1:1787761728.199168:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945f00.
00010000:00000010:1.1:1787761728.199170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b0800.
00010000:00000010:1.1:1787761728.199172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945c00.
00010000:00000010:1.1:1787761728.199174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b1000.
00010000:00000010:1.1:1787761728.199177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03c00.
00010000:00000010:1.1:1787761728.199179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7834400.
00010000:00000010:1.1:1787761728.199181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2c00.
00010000:00000010:1.1:1787761728.199183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7834e00.
00010000:00000010:1.1:1787761728.199185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945800.
00010000:00000010:1.1:1787761728.199187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b1800.
00010000:00000010:1.1:1787761728.199189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b100.
00010000:00000010:1.1:1787761728.199191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f84ac600.
00010000:00000010:1.1:1787761728.199192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f128b600.
00010000:00000010:1.1:1787761728.199194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f87b0a00.
00000800:00000200:2.1:1787761730.621213:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761730.621221:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787761730.622473:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761730.622476:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787761730.622913:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761730.622916:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787761730.623410:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761730.623414:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787761735.741532:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761735.741535:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787761735.741796:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761735.741798:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787761735.742045:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761735.742048:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787761735.742285:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761735.742288:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00010000:00000010:3.1:1787761737.411169:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03400.
00010000:00000010:3.1:1787761737.411177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873a000.
00010000:00000010:3.1:1787761737.411181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac200.
00010000:00000010:3.1:1787761737.411184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873e000.
00010000:00000010:3.1:1787761737.411186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac400.
00010000:00000010:3.1:1787761737.411188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873f600.
00010000:00000010:3.1:1787761737.411191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac900.
00010000:00000010:3.1:1787761737.411193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7842400.
00010000:00000010:3.1:1787761737.411195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03900.
00010000:00000010:3.1:1787761737.411198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873a400.
00010000:00000010:3.1:1787761737.411200:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2100.
00010000:00000010:3.1:1787761737.411202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873e200.
00010000:00000010:3.1:1787761737.411203:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2400.
00010000:00000010:3.1:1787761737.411205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79f2c00.
00010000:00000010:3.1:1787761737.411207:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87acf00.
00010000:00000010:3.1:1787761737.411223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873ae00.
00010000:00000010:3.1:1787761737.411225:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2900.
00010000:00000010:3.1:1787761737.411228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7834800.
00010000:00000010:3.1:1787761737.411230:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87acc00.
00010000:00000010:3.1:1787761737.411232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873a800.
00010000:00000010:3.1:1787761737.411234:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945000.
00010000:00000010:3.1:1787761737.411236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873b200.
00010000:00000010:3.1:1787761737.411238:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945300.
00010000:00000010:3.1:1787761737.411240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843200.
00010000:00000010:3.1:1787761737.411241:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03500.
00010000:00000010:3.1:1787761737.411243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7834200.
00010000:00000010:3.1:1787761737.411245:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945e00.
00010000:00000010:3.1:1787761737.411247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7842600.
00010000:00000010:3.1:1787761737.411249:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945600.
00010000:00000010:3.1:1787761737.411251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843000.
00010000:00000010:3.1:1787761737.411253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d7945d00.
00010000:00000010:3.1:1787761737.411255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7842e00.
00000800:00000200:2.1:1787761740.861329:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761740.861358:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787761740.862669:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761740.862674:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787761740.862987:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761740.862990:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787761740.863331:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761740.863333:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787761745.981503:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761745.981512:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787761745.982581:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761745.982584:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787761745.983043:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761745.983047:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787761745.983636:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761745.983642:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00010000:00000010:1.1:1787761747.652203:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88aee00.
00010000:00000010:1.1:1787761747.652215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867c800.
00010000:00000010:1.1:1787761747.652221:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03000.
00010000:00000010:1.1:1787761747.652226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873e800.
00010000:00000010:1.1:1787761747.652231:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5ad00.
00010000:00000010:1.1:1787761747.652235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873ec00.
00010000:00000010:1.1:1787761747.652240:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac000.
00010000:00000010:1.1:1787761747.652244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873ee00.
00010000:00000010:1.1:1787761747.652247:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88aeb00.
00010000:00000010:1.1:1787761747.652251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873e400.
00010000:00000010:1.1:1787761747.652254:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac500.
00010000:00000010:1.1:1787761747.652258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873fe00.
00010000:00000010:1.1:1787761747.652262:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac300.
00010000:00000010:1.1:1787761747.652265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873f200.
00010000:00000010:1.1:1787761747.652269:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac100.
00010000:00000010:1.1:1787761747.652272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843400.
00010000:00000010:1.1:1787761747.652276:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87acb00.
00010000:00000010:1.1:1787761747.652279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7842800.
00010000:00000010:1.1:1787761747.652283:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae300.
00010000:00000010:1.1:1787761747.652286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873f400.
00010000:00000010:1.1:1787761747.652289:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ace00.
00010000:00000010:1.1:1787761747.652293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873fc00.
00010000:00000010:1.1:1787761747.656201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87acd00.
00010000:00000010:1.1:1787761747.656212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873fa00.
00010000:00000010:1.1:1787761747.656217:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03100.
00010000:00000010:1.1:1787761747.656221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7940000.
00010000:00000010:1.1:1787761747.656225:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87aca00.
00010000:00000010:1.1:1787761747.656229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873e600.
00010000:00000010:1.1:1787761747.656233:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88aea00.
00010000:00000010:1.1:1787761747.656236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873f000.
00010000:00000010:1.1:1787761747.656239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac800.
00010000:00000010:1.1:1787761747.656243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867d400.
00010000:00000010:1.1:1787761747.656247:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac600.
00010000:00000010:1.1:1787761747.656250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873ea00.
00010000:00000010:1.1:1787761747.656253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87ac700.
00010000:00000010:1.1:1787761747.656256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873f800.
00000800:00000200:2.1:1787761751.101376:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761751.101389:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787761751.102214:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761751.102217:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787761751.102581:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761751.102584:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787761751.103018:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761751.103022:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787761756.221177:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761756.221184:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787761756.222283:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761756.222287:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787761756.222743:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761756.222747:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787761756.223103:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761756.223106:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00010000:00000010:0.1:1787761757.892195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5a800.
00010000:00000010:0.1:1787761757.892206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780de00.
00010000:00000010:0.1:1787761757.892281:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5a300.
00010000:00000010:0.1:1787761757.892286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7842000.
00010000:00000010:0.1:1787761757.892291:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae400.
00010000:00000010:0.1:1787761757.892295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843c00.
00010000:00000010:0.1:1787761757.892298:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae000.
00010000:00000010:0.1:1787761757.892302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843e00.
00010000:00000010:0.1:1787761757.896187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae600.
00010000:00000010:0.1:1787761757.896195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867ca00.
00010000:00000010:0.1:1787761757.896200:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2200.
00010000:00000010:0.1:1787761757.896204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780dc00.
00010000:00000010:0.1:1787761757.896207:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2000.
00010000:00000010:0.1:1787761757.896212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780cc00.
00010000:00000010:0.1:1787761757.896215:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae800.
00010000:00000010:0.1:1787761757.896219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867de00.
00010000:00000010:0.1:1787761757.896222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88aef00.
00010000:00000010:0.1:1787761757.896225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780d400.
00010000:00000010:0.1:1787761757.896229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5a600.
00010000:00000010:0.1:1787761757.896233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7842c00.
00010000:00000010:0.1:1787761757.896236:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5a700.
00010000:00000010:0.1:1787761757.896239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7842a00.
00010000:00000010:0.1:1787761757.896243:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03700.
00010000:00000010:0.1:1787761757.896246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780d600.
00010000:00000010:0.1:1787761757.896249:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae900.
00010000:00000010:0.1:1787761757.896252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843800.
00010000:00000010:0.1:1787761757.896255:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5a100.
00010000:00000010:0.1:1787761757.896259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7940c00.
00010000:00000010:0.1:1787761757.896262:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03a00.
00010000:00000010:0.1:1787761757.896265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7941e00.
00010000:00000010:0.1:1787761757.896268:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5a900.
00010000:00000010:0.1:1787761757.896272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8791800.
00010000:00000010:0.1:1787761757.896284:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae500.
00010000:00000010:0.1:1787761757.896288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867d200.
00000800:00000200:2.1:1787761761.340745:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761761.341054:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761761.341220:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761761.341358:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761766.462065:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761766.462077:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787761766.462935:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761766.462938:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787761766.463637:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761766.463641:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787761766.464109:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761766.464113:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00010000:00000010:0.1:1787761768.132213:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5a400.
00010000:00000010:0.1:1787761768.132228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797e800.
00010000:00000010:0.1:1787761768.132236:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518da00.
00010000:00000010:0.1:1787761768.132245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7940e00.
00010000:00000010:0.1:1787761768.132250:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c518d700.
00010000:00000010:0.1:1787761768.132256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867da00.
00010000:00000010:0.1:1787761768.132262:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5a000.
00010000:00000010:0.1:1787761768.132268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797f000.
00010000:00000010:0.1:1787761768.132275:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03600.
00010000:00000010:0.1:1787761768.132280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7940400.
00010000:00000010:0.1:1787761768.132287:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03b00.
00010000:00000010:0.1:1787761768.132294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7940200.
00010000:00000010:0.1:1787761768.132299:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae200.
00010000:00000010:0.1:1787761768.132305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867c400.
00010000:00000010:0.1:1787761768.132309:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88aec00.
00010000:00000010:0.1:1787761768.132313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780da00.
00010000:00000010:0.1:1787761768.132316:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5ae00.
00010000:00000010:0.1:1787761768.132320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797f400.
00010000:00000010:0.1:1787761768.132323:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5af00.
00010000:00000010:0.1:1787761768.136210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7842200.
00010000:00000010:0.1:1787761768.136224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03f00.
00010000:00000010:0.1:1787761768.136231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7940a00.
00010000:00000010:0.1:1787761768.136237:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88ae700.
00010000:00000010:0.1:1787761768.136242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867c600.
00010000:00000010:0.1:1787761768.136249:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f88aed00.
00010000:00000010:0.1:1787761768.136254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867c000.
00010000:00000010:0.1:1787761768.136260:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03d00.
00010000:00000010:0.1:1787761768.136266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843600.
00010000:00000010:0.1:1787761768.136272:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5ac00.
00010000:00000010:0.1:1787761768.136278:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7843a00.
00010000:00000010:0.1:1787761768.136283:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5ab00.
00010000:00000010:0.1:1787761768.136289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780ca00.
00000800:00000200:1.1:1787761771.581308:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761771.581322:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787761771.582254:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761771.582257:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787761771.582651:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761771.582654:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787761771.583109:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761771.583112:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787761776.701424:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761776.701437:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787761776.702404:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761776.702408:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787761776.702874:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761776.702877:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787761776.703270:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761776.703273:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00010000:00000010:0.1:1787761777.348223:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d100.
00010000:00000010:0.1:1787761777.348235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867d000.
00010000:00000010:0.1:1787761777.348239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6db00.
00010000:00000010:0.1:1787761777.348243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780c600.
00010000:00000010:0.1:1787761777.348248:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5800.
00010000:00000010:0.1:1787761777.348252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797ec00.
00010000:00000010:0.1:1787761777.348256:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7800.
00010000:00000010:0.1:1787761777.348259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7940800.
00010000:00000010:0.1:1787761777.348263:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5600.
00010000:00000010:0.1:1787761777.348267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797f800.
00010000:00000010:0.1:1787761777.348270:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5a00.
00010000:00000010:0.1:1787761777.348275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780d200.
00010000:00000010:0.1:1787761777.348278:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6da00.
00010000:00000010:0.1:1787761777.348282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780ce00.
00010000:00000010:0.1:1787761777.348285:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6dd00.
00010000:00000010:0.1:1787761777.348288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867d800.
00010000:00000010:0.1:1787761777.352220:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5700.
00010000:00000010:0.1:1787761777.352231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780c800.
00010000:00000010:0.1:1787761777.352236:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2e00.
00010000:00000010:0.1:1787761777.352239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797ea00.
00010000:00000010:0.1:1787761777.352243:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5200.
00010000:00000010:0.1:1787761777.352246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797fe00.
00010000:00000010:0.1:1787761777.352250:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa03200.
00010000:00000010:0.1:1787761777.352253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780d800.
00010000:00000010:0.1:1787761777.352257:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5a200.
00010000:00000010:0.1:1787761777.352261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797f600.
00010000:00000010:0.1:1787761777.352264:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c3d90c00.
00010000:00000010:0.1:1787761777.352268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867dc00.
00010000:00000010:0.1:1787761777.352271:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5aa00.
00010000:00000010:0.1:1787761777.352275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7941000.
00010000:00000010:0.1:1787761777.352278:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2700.
00010000:00000010:0.1:1787761777.352281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780d000.
00010000:00000010:0.1:1787761777.352285:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751c8c5a500.
00010000:00000010:0.1:1787761777.352288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7941a00.
00000800:00000200:2.1:1787761781.821176:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761781.821181:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787761781.822597:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761781.822599:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787761781.822847:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761781.822849:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787761781.823115:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761781.823119:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787761786.941442:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761786.941453:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787761786.942371:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761786.942374:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787761786.942774:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761786.942778:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787761786.943121:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761786.943124:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00010000:00000010:2.1:1787761787.587157:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f200.
00010000:00000010:2.1:1787761787.587165:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3da00.
00010000:00000010:2.1:1787761787.587228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5300.
00010000:00000010:2.1:1787761787.587232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797e400.
00010000:00000010:2.1:1787761787.587235:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f100.
00010000:00000010:2.1:1787761787.587238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7941800.
00010000:00000010:2.1:1787761787.587241:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e400.
00010000:00000010:2.1:1787761787.587282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867c200.
00010000:00000010:2.1:1787761787.587285:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7e00.
00010000:00000010:2.1:1787761787.587287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7940600.
00010000:00000010:2.1:1787761787.587289:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5d00.
00010000:00000010:2.1:1787761787.587291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3cc00.
00010000:00000010:2.1:1787761787.587293:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5900.
00010000:00000010:2.1:1787761787.587295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3ca00.
00010000:00000010:2.1:1787761787.587296:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4ef00.
00010000:00000010:2.1:1787761787.587299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867ce00.
00010000:00000010:2.1:1787761787.587301:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7c00.
00010000:00000010:2.1:1787761787.587303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3d800.
00010000:00000010:2.1:1787761787.587305:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5500.
00010000:00000010:2.1:1787761787.587307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797e600.
00010000:00000010:2.1:1787761787.587309:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d800.
00010000:00000010:2.1:1787761787.587311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7941200.
00010000:00000010:2.1:1787761787.587312:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5f00.
00010000:00000010:2.1:1787761787.587314:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780c400.
00010000:00000010:2.1:1787761787.587316:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6df00.
00010000:00000010:2.1:1787761787.587318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797fc00.
00010000:00000010:2.1:1787761787.587319:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5e00.
00010000:00000010:2.1:1787761787.587321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780c000.
00010000:00000010:2.1:1787761787.587322:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5c00.
00010000:00000010:2.1:1787761787.587324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d780c200.
00010000:00000010:2.1:1787761787.587326:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dac6d600.
00010000:00000010:2.1:1787761787.587328:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867d600.
00000800:00000200:1.1:1787761792.061472:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761792.061488:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787761792.062375:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761792.062379:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787761792.062888:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761792.062894:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787761792.063384:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761792.063387:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787761797.181218:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761797.181228:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1787761797.181810:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761797.181813:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1787761797.182103:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761797.182106:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1787761797.182420:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761797.182423:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00010000:00000010:1.1:1787761797.827162:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e700.
00010000:00000010:1.1:1787761797.827169:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f867cc00.
00010000:00000010:1.1:1787761797.827173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7000.
00010000:00000010:1.1:1787761797.827175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79af600.
00010000:00000010:1.1:1787761797.827218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7200.
00010000:00000010:1.1:1787761797.827220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79af200.
00010000:00000010:1.1:1787761797.827222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870af00.
00010000:00000010:1.1:1787761797.827250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3c000.
00010000:00000010:1.1:1787761797.827252:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7400.
00010000:00000010:1.1:1787761797.827254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79ae800.
00010000:00000010:1.1:1787761797.827256:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7900.
00010000:00000010:1.1:1787761797.827258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f873a200.
00010000:00000010:1.1:1787761797.827264:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870ac00.
00010000:00000010:1.1:1787761797.827266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3ce00.
00010000:00000010:1.1:1787761797.827268:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751cffa2800.
00010000:00000010:1.1:1787761797.827270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3c800.
00010000:00000010:1.1:1787761797.827272:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5100.
00010000:00000010:1.1:1787761797.827275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797e200.
00010000:00000010:1.1:1787761797.827277:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5b00.
00010000:00000010:1.1:1787761797.827279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797f200.
00010000:00000010:1.1:1787761797.827280:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7500.
00010000:00000010:1.1:1787761797.827282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7941400.
00010000:00000010:1.1:1787761797.827285:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5000.
00010000:00000010:1.1:1787761797.827287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3d200.
00010000:00000010:1.1:1787761797.827289:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d79c5400.
00010000:00000010:1.1:1787761797.827291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3de00.
00010000:00000010:1.1:1787761797.827293:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f600.
00010000:00000010:1.1:1787761797.827295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3dc00.
00010000:00000010:1.1:1787761797.827296:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0fa00.
00010000:00000010:1.1:1787761797.827298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7941c00.
00010000:00000010:1.1:1787761797.827299:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f000.
00010000:00000010:1.1:1787761797.827301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3c400.
00000800:00000200:1.1:1787761802.301874:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761802.301879:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787761802.302515:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761802.302521:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787761802.302950:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761802.302954:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787761802.303469:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761802.303473:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787761807.421252:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761807.421266:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787761807.422318:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761807.422344:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787761807.422731:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761807.422734:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787761807.423126:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761807.423129:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00010000:00000010:2.1:1787761808.067172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7600.
00010000:00000010:2.1:1787761808.067179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b8c00.
00010000:00000010:2.1:1787761808.067183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a900.
00010000:00000010:2.1:1787761808.067185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3c200.
00010000:00000010:2.1:1787761808.067188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0ff00.
00010000:00000010:2.1:1787761808.067190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ea00.
00010000:00000010:2.1:1787761808.067192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870aa00.
00010000:00000010:2.1:1787761808.067194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d7941600.
00010000:00000010:2.1:1787761808.067213:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a600.
00010000:00000010:2.1:1787761808.067215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3d400.
00010000:00000010:2.1:1787761808.067217:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870ab00.
00010000:00000010:2.1:1787761808.067220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797e000.
00010000:00000010:2.1:1787761808.067222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7300.
00010000:00000010:2.1:1787761808.067224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aec00.
00010000:00000010:2.1:1787761808.067226:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7a00.
00010000:00000010:2.1:1787761808.067229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79afc00.
00010000:00000010:2.1:1787761808.067231:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a300.
00010000:00000010:2.1:1787761808.067233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797fa00.
00010000:00000010:2.1:1787761808.067235:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0fc00.
00010000:00000010:2.1:1787761808.067237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3c600.
00010000:00000010:2.1:1787761808.067239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0fb00.
00010000:00000010:2.1:1787761808.067240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79ae000.
00010000:00000010:2.1:1787761808.067242:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a700.
00010000:00000010:2.1:1787761808.067243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3d000.
00010000:00000010:2.1:1787761808.067245:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a200.
00010000:00000010:2.1:1787761808.067247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c7b3d600.
00010000:00000010:2.1:1787761808.067248:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e300.
00010000:00000010:2.1:1787761808.067250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ec00.
00010000:00000010:2.1:1787761808.067252:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a400.
00010000:00000010:2.1:1787761808.067253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d797ee00.
00010000:00000010:2.1:1787761808.067255:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e600.
00010000:00000010:2.1:1787761808.067256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d1d49800.
00000800:00000200:1.1:1787761812.540795:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761812.540798:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787761812.541303:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761812.541305:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787761812.541525:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761812.541527:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787761812.541704:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761812.541706:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787761817.661687:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761817.661699:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787761817.662976:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761817.662979:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787761817.663419:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761817.663423:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787761817.663778:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761817.663782:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00010000:00000010:0.1:1787761818.307218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e100.
00010000:00000010:0.1:1787761818.307232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f59e00.
00010000:00000010:0.1:1787761818.307237:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0fd00.
00010000:00000010:0.1:1787761818.307242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b9400.
00010000:00000010:0.1:1787761818.307269:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f500.
00010000:00000010:0.1:1787761818.307276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aee00.
00010000:00000010:0.1:1787761818.307283:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2200.
00010000:00000010:0.1:1787761818.307287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ff200.
00010000:00000010:0.1:1787761818.307292:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870ad00.
00010000:00000010:0.1:1787761818.307297:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ee00.
00010000:00000010:0.1:1787761818.307301:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a800.
00010000:00000010:0.1:1787761818.307305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b9200.
00010000:00000010:0.1:1787761818.307308:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a000.
00010000:00000010:0.1:1787761818.307312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b9600.
00010000:00000010:0.1:1787761818.307315:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a100.
00010000:00000010:0.1:1787761818.307318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ffa00.
00010000:00000010:0.1:1787761818.307321:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e000.
00010000:00000010:0.1:1787761818.307325:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539f000.
00010000:00000010:0.1:1787761818.307328:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e800.
00010000:00000010:0.1:1787761818.307331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ff400.
00010000:00000010:0.1:1787761818.307334:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870a500.
00010000:00000010:0.1:1787761818.307337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b8400.
00010000:00000010:0.1:1787761818.312198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7b00.
00010000:00000010:0.1:1787761818.312210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79afa00.
00010000:00000010:0.1:1787761818.312215:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4ec00.
00010000:00000010:0.1:1787761818.312218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539fa00.
00010000:00000010:0.1:1787761818.312222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7f00.
00010000:00000010:0.1:1787761818.312225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79af400.
00010000:00000010:0.1:1787761818.312228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f870ae00.
00010000:00000010:0.1:1787761818.312232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ff800.
00010000:00000010:0.1:1787761818.312235:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7d00.
00010000:00000010:0.1:1787761818.312238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79ae400.
00010000:00000010:0.1:1787761818.312242:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7700.
00010000:00000010:0.1:1787761818.312266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79ae200.
00000800:00000200:1.1:1787761822.781305:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761822.781313:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787761822.782458:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761822.782464:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787761822.782895:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761822.782900:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787761822.783330:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761822.783334:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787761827.900899:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761827.901326:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761827.901521:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761827.901710:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00010000:00000010:2.1:1787761828.548229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f800.
00010000:00000010:2.1:1787761828.548238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b8200.
00010000:00000010:2.1:1787761828.548242:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78f7e00.
00010000:00000010:2.1:1787761828.548246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79ae600.
00010000:00000010:2.1:1787761828.548251:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4ea00.
00010000:00000010:2.1:1787761828.548254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79af000.
00010000:00000010:2.1:1787761828.548258:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2900.
00010000:00000010:2.1:1787761828.548262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79af800.
00010000:00000010:2.1:1787761828.548265:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2500.
00010000:00000010:2.1:1787761828.548269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b8600.
00010000:00000010:2.1:1787761828.548272:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2e00.
00010000:00000010:2.1:1787761828.548275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b8800.
00010000:00000010:2.1:1787761828.548278:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f900.
00010000:00000010:2.1:1787761828.548281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79aea00.
00010000:00000010:2.1:1787761828.548284:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e900.
00010000:00000010:2.1:1787761828.548289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440a00.
00010000:00000010:2.1:1787761828.548292:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78f7b00.
00010000:00000010:2.1:1787761828.548295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751d79afe00.
00010000:00000010:2.1:1787761828.548299:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2b00.
00010000:00000010:2.1:1787761828.548302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ffc00.
00010000:00000010:2.1:1787761828.548306:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2800.
00010000:00000010:2.1:1787761828.548309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b9e00.
00010000:00000010:2.1:1787761828.548312:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4ee00.
00010000:00000010:2.1:1787761828.548316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441000.
00010000:00000010:2.1:1787761828.548319:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2600.
00010000:00000010:2.1:1787761828.548322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fe000.
00010000:00000010:2.1:1787761828.548325:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f87a7100.
00010000:00000010:2.1:1787761828.548328:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b9a00.
00010000:00000010:2.1:1787761828.548330:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2a00.
00010000:00000010:2.1:1787761828.548333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ff600.
00010000:00000010:2.1:1787761828.548336:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4ed00.
00010000:00000010:2.1:1787761828.548339:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c6f58e00.
00000800:00000200:1.1:1787761833.021460:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761833.021475:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787761833.022476:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761833.022480:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787761833.022953:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761833.022957:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787761833.023486:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761833.023489:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787761838.140846:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761838.140852:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787761838.141491:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761838.141492:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787761838.141681:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761838.141682:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787761838.141863:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761838.141865:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00010000:00000010:2.1:1787761838.788214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78f7900.
00010000:00000010:2.1:1787761838.788224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86bae00.
00010000:00000010:2.1:1787761838.788228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2100.
00010000:00000010:2.1:1787761838.788230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c0800.
00010000:00000010:2.1:1787761838.788233:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f400.
00010000:00000010:2.1:1787761838.788236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c1400.
00010000:00000010:2.1:1787761838.788238:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2000.
00010000:00000010:2.1:1787761838.788241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b9c00.
00010000:00000010:2.1:1787761838.788243:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9a00.
00010000:00000010:2.1:1787761838.788247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b8e00.
00010000:00000010:2.1:1787761838.788249:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78f7d00.
00010000:00000010:2.1:1787761838.788251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86bb400.
00010000:00000010:2.1:1787761838.788253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f700.
00010000:00000010:2.1:1787761838.788255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86bb800.
00010000:00000010:2.1:1787761838.788257:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2300.
00010000:00000010:2.1:1787761838.788260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b9800.
00010000:00000010:2.1:1787761838.788262:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0f300.
00010000:00000010:2.1:1787761838.788264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ba200.
00010000:00000010:2.1:1787761838.788267:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2c00.
00010000:00000010:2.1:1787761838.788269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b8a00.
00010000:00000010:2.1:1787761838.788271:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e500.
00010000:00000010:2.1:1787761838.788273:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b8000.
00010000:00000010:2.1:1787761838.788275:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4eb00.
00010000:00000010:2.1:1787761838.788277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86b9000.
00010000:00000010:2.1:1787761838.788279:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78f7500.
00010000:00000010:2.1:1787761838.788281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86baa00.
00010000:00000010:2.1:1787761838.788283:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2d00.
00010000:00000010:2.1:1787761838.788285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fe600.
00010000:00000010:2.1:1787761838.788287:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78f7f00.
00010000:00000010:2.1:1787761838.788289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86bac00.
00010000:00000010:2.1:1787761838.792187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0c4e200.
00010000:00000010:2.1:1787761838.792194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ffe00.
00000800:00000200:1.1:1787761843.260959:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761843.260969:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787761843.261707:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761843.261709:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787761843.261915:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761843.261916:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787761843.262120:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761843.262122:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787761848.381288:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761848.381303:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787761848.382130:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761848.382134:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787761848.382645:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761848.382650:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787761848.383084:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761848.383088:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00010000:00000010:2.1:1787761849.028205:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6fc00.
00010000:00000010:2.1:1787761849.028217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fec00.
00010000:00000010:2.1:1787761849.028222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78f7000.
00010000:00000010:2.1:1787761849.028227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ba000.
00010000:00000010:2.1:1787761849.028230:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9200.
00010000:00000010:2.1:1787761849.028233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843e800.
00010000:00000010:2.1:1787761849.028237:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6fb00.
00010000:00000010:2.1:1787761849.028240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c0a00.
00010000:00000010:2.1:1787761849.028243:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78f7200.
00010000:00000010:2.1:1787761849.028246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86bb200.
00010000:00000010:2.1:1787761849.028249:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9900.
00010000:00000010:2.1:1787761849.028252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843e000.
00010000:00000010:2.1:1787761849.028255:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6f800.
00010000:00000010:2.1:1787761849.028258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ba600.
00010000:00000010:2.1:1787761849.032220:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9f00.
00010000:00000010:2.1:1787761849.032232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe440000.
00010000:00000010:2.1:1787761849.032237:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6f600.
00010000:00000010:2.1:1787761849.032242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c0200.
00010000:00000010:2.1:1787761849.032246:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78f7400.
00010000:00000010:2.1:1787761849.032251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c0e00.
00010000:00000010:2.1:1787761849.032254:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6fa00.
00010000:00000010:2.1:1787761849.032258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86bb000.
00010000:00000010:2.1:1787761849.032262:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2700.
00010000:00000010:2.1:1787761849.032266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86bba00.
00010000:00000010:2.1:1787761849.032269:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2f00.
00010000:00000010:2.1:1787761849.032273:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fee00.
00010000:00000010:2.1:1787761849.032276:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fa7c2400.
00010000:00000010:2.1:1787761849.032279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fe200.
00010000:00000010:2.1:1787761849.032283:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9800.
00010000:00000010:2.1:1787761849.032286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441a00.
00010000:00000010:2.1:1787761849.032289:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9300.
00010000:00000010:2.1:1787761849.032292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fe441400.
00000800:00000200:1.1:1787761853.501344:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761853.501353:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787761853.502164:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761853.502168:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787761853.502536:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761853.502539:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787761853.502878:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761853.502881:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787761858.620871:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761858.620877:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787761858.621441:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761858.621444:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787761858.621787:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761858.621790:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787761858.622118:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761858.622121:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00010000:00000010:2.1:1787761859.267166:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6f00.
00010000:00000010:2.1:1787761859.267174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c2400.
00010000:00000010:2.1:1787761859.267177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6e00.
00010000:00000010:2.1:1787761859.267179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c3800.
00010000:00000010:2.1:1787761859.267182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9100.
00010000:00000010:2.1:1787761859.267184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c2200.
00010000:00000010:2.1:1787761859.267186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9d00.
00010000:00000010:2.1:1787761859.267188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c2a00.
00010000:00000010:2.1:1787761859.271166:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9000.
00010000:00000010:2.1:1787761859.271172:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c2c00.
00010000:00000010:2.1:1787761859.271175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78bea00.
00010000:00000010:2.1:1787761859.271177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec04e00.
00010000:00000010:2.1:1787761859.271180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6f100.
00010000:00000010:2.1:1787761859.271182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c1600.
00010000:00000010:2.1:1787761859.271184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9700.
00010000:00000010:2.1:1787761859.271186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c3c00.
00010000:00000010:2.1:1787761859.271187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6f700.
00010000:00000010:2.1:1787761859.271189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec04400.
00010000:00000010:2.1:1787761859.271191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9600.
00010000:00000010:2.1:1787761859.271193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c2000.
00010000:00000010:2.1:1787761859.271194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9b00.
00010000:00000010:2.1:1787761859.271196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c3600.
00010000:00000010:2.1:1787761859.271197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6f300.
00010000:00000010:2.1:1787761859.271199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec04200.
00010000:00000010:2.1:1787761859.271201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6fd00.
00010000:00000010:2.1:1787761859.271202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec04a00.
00010000:00000010:2.1:1787761859.271204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6f400.
00010000:00000010:2.1:1787761859.271205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec04c00.
00010000:00000010:2.1:1787761859.271207:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9c00.
00010000:00000010:2.1:1787761859.271209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c3200.
00010000:00000010:2.1:1787761859.271210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9e00.
00010000:00000010:2.1:1787761859.271212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c2600.
00010000:00000010:2.1:1787761859.271213:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9500.
00010000:00000010:2.1:1787761859.271215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c2800.
00010000:00000010:2.1:1787761859.271216:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6f000.
00010000:00000010:2.1:1787761859.271218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fea00.
00010000:00000010:2.1:1787761859.271220:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6f200.
00010000:00000010:2.1:1787761859.271222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ff000.
00010000:00000010:2.1:1787761859.271224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59bb500.
00010000:00000010:2.1:1787761859.271226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c0600.
00010000:00000010:2.1:1787761859.271228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78f7600.
00010000:00000010:2.1:1787761859.271231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c1c00.
00010000:00000010:2.1:1787761859.271232:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6f900.
00010000:00000010:2.1:1787761859.271234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86ba800.
00010000:00000010:2.1:1787761859.271236:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6f500.
00010000:00000010:2.1:1787761859.271237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fe400.
00010000:00000010:2.1:1787761859.271239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59bbe00.
00010000:00000010:2.1:1787761859.271241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c1800.
00010000:00000010:2.1:1787761859.271243:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6ff00.
00010000:00000010:2.1:1787761859.271244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86fe800.
00010000:00000010:2.1:1787761859.271246:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59bbb00.
00010000:00000010:2.1:1787761859.271247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c1a00.
00010000:00000010:2.1:1787761859.271249:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59bb300.
00010000:00000010:2.1:1787761859.271250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c1200.
00010000:00000010:2.1:1787761859.271252:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfa6fe00.
00010000:00000010:2.1:1787761859.271253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c0000.
00010000:00000010:2.1:1787761859.271255:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59bb600.
00010000:00000010:2.1:1787761859.271257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843f000.
00010000:00000010:2.1:1787761859.271258:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d0db9400.
00010000:00000010:2.1:1787761859.271260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f843e400.
00010000:00000010:2.1:1787761859.271261:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78f7a00.
00010000:00000010:2.1:1787761859.271263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f86bb600.
00010000:00000010:2.1:1787761859.271265:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751fdd0fe00.
00010000:00000010:2.1:1787761859.271277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c1e00.
00000800:00000200:1.1:1787761863.740805:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761863.740811:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787761863.741297:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761863.741299:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787761863.741508:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761863.741509:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787761863.741692:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761863.741694:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00010000:00000010:2.1:1787761868.485162:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780300.
00010000:00000010:2.1:1787761868.485170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8797400.
00010000:00000010:2.1:1787761868.485173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780600.
00010000:00000010:2.1:1787761868.485176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8796400.
00010000:00000010:2.1:1787761868.485179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780a00.
00010000:00000010:2.1:1787761868.485181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8797800.
00010000:00000010:2.1:1787761868.485183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780700.
00010000:00000010:2.1:1787761868.485185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8796200.
00010000:00000010:2.1:1787761868.489163:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780000.
00010000:00000010:2.1:1787761868.489169:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8796a00.
00010000:00000010:2.1:1787761868.489171:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780200.
00010000:00000010:2.1:1787761868.489173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8796c00.
00010000:00000010:2.1:1787761868.489176:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6400.
00010000:00000010:2.1:1787761868.489179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8797e00.
00010000:00000010:2.1:1787761868.489181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6900.
00010000:00000010:2.1:1787761868.489183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8797c00.
00010000:00000010:2.1:1787761868.489185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6d00.
00010000:00000010:2.1:1787761868.489187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8796000.
00010000:00000010:2.1:1787761868.489188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6500.
00010000:00000010:2.1:1787761868.489190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8797600.
00010000:00000010:2.1:1787761868.489192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6f00.
00010000:00000010:2.1:1787761868.489193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8797200.
00010000:00000010:2.1:1787761868.489195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6e00.
00010000:00000010:2.1:1787761868.489196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8796600.
00010000:00000010:2.1:1787761868.489198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6c00.
00010000:00000010:2.1:1787761868.489200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8796800.
00010000:00000010:2.1:1787761868.489201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6b00.
00010000:00000010:2.1:1787761868.489203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8797000.
00010000:00000010:2.1:1787761868.489204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6800.
00010000:00000010:2.1:1787761868.489206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8797a00.
00010000:00000010:2.1:1787761868.489208:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6300.
00010000:00000010:2.1:1787761868.489209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8794000.
00010000:00000010:2.1:1787761868.489212:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6600.
00010000:00000010:2.1:1787761868.489213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8794200.
00010000:00000010:2.1:1787761868.489215:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6a00.
00010000:00000010:2.1:1787761868.489217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8794a00.
00010000:00000010:2.1:1787761868.489220:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6700.
00010000:00000010:2.1:1787761868.489222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8794c00.
00010000:00000010:2.1:1787761868.489224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6000.
00010000:00000010:2.1:1787761868.489226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8795e00.
00010000:00000010:2.1:1787761868.489228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6200.
00010000:00000010:2.1:1787761868.489230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8795c00.
00010000:00000010:2.1:1787761868.489231:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f59a6100.
00010000:00000010:2.1:1787761868.489233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8795600.
00010000:00000010:2.1:1787761868.489235:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78be800.
00010000:00000010:2.1:1787761868.489236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec04600.
00010000:00000010:2.1:1787761868.489239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49a00.
00010000:00000010:2.1:1787761868.489241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8795200.
00010000:00000010:2.1:1787761868.489243:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49700.
00010000:00000010:2.1:1787761868.489244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8794600.
00010000:00000010:2.1:1787761868.489246:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49000.
00010000:00000010:2.1:1787761868.489248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8794800.
00010000:00000010:2.1:1787761868.489249:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49200.
00010000:00000010:2.1:1787761868.489251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8795000.
00010000:00000010:2.1:1787761868.489253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49100.
00010000:00000010:2.1:1787761868.489255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8795a00.
00010000:00000010:2.1:1787761868.489257:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49400.
00010000:00000010:2.1:1787761868.489259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8794e00.
00010000:00000010:2.1:1787761868.489261:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49900.
00010000:00000010:2.1:1787761868.489263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8795400.
00010000:00000010:2.1:1787761868.489264:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49d00.
00010000:00000010:2.1:1787761868.489266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8794400.
00010000:00000010:2.1:1787761868.489267:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49500.
00010000:00000010:2.1:1787761868.489269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8795800.
00010000:00000010:2.1:1787761868.489271:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49f00.
00010000:00000010:2.1:1787761868.489273:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec07600.
00010000:00000010:2.1:1787761868.489275:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49e00.
00010000:00000010:2.1:1787761868.489276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec07200.
00010000:00000010:2.1:1787761868.489279:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49c00.
00010000:00000010:2.1:1787761868.489281:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec06600.
00010000:00000010:2.1:1787761868.489283:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49b00.
00010000:00000010:2.1:1787761868.489285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec06800.
00010000:00000010:2.1:1787761868.489287:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49800.
00010000:00000010:2.1:1787761868.489289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec07000.
00010000:00000010:2.1:1787761868.489291:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49300.
00010000:00000010:2.1:1787761868.489292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec07a00.
00010000:00000010:2.1:1787761868.489294:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751e0f49600.
00010000:00000010:2.1:1787761868.489295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec06e00.
00010000:00000010:2.1:1787761868.489297:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6c00.
00010000:00000010:2.1:1787761868.489299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec07400.
00010000:00000010:2.1:1787761868.489301:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6b00.
00010000:00000010:2.1:1787761868.489302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec06400.
00010000:00000010:2.1:1787761868.489304:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6800.
00010000:00000010:2.1:1787761868.489305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec07800.
00010000:00000010:2.1:1787761868.489307:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6300.
00010000:00000010:2.1:1787761868.489309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec06200.
00010000:00000010:2.1:1787761868.489310:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6600.
00010000:00000010:2.1:1787761868.489312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec06a00.
00010000:00000010:2.1:1787761868.489313:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6a00.
00010000:00000010:2.1:1787761868.489315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec06c00.
00010000:00000010:2.1:1787761868.489317:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6700.
00010000:00000010:2.1:1787761868.489318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec07e00.
00010000:00000010:2.1:1787761868.489320:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6000.
00010000:00000010:2.1:1787761868.489321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec07c00.
00010000:00000010:2.1:1787761868.489323:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6200.
00010000:00000010:2.1:1787761868.489325:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec06000.
00010000:00000010:2.1:1787761868.489326:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6100.
00010000:00000010:2.1:1787761868.489328:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c3000.
00010000:00000010:2.1:1787761868.489330:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6400.
00010000:00000010:2.1:1787761868.489331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c3e00.
00010000:00000010:2.1:1787761868.489334:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6900.
00010000:00000010:2.1:1787761868.489336:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c3a00.
00010000:00000010:2.1:1787761868.489338:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6d00.
00010000:00000010:2.1:1787761868.489340:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c2e00.
00010000:00000010:2.1:1787761868.489342:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfbe6500.
00010000:00000010:2.1:1787761868.489343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9752011c3400.
00000800:00000200:1.1:1787761868.861319:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761868.861323:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787761868.861672:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761868.861674:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787761868.861895:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761868.861910:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787761868.862109:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761868.862112:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787761873.980832:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761873.980839:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787761873.981372:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761873.981373:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787761873.981615:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761873.981617:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787761873.981810:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761873.981812:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00010000:00000010:2.1:1787761878.725167:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78be300.
00010000:00000010:2.1:1787761878.725175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f861ea00.
00010000:00000010:2.1:1787761878.725178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78beb00.
00010000:00000010:2.1:1787761878.725180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f861ec00.
00010000:00000010:2.1:1787761878.725182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78be700.
00010000:00000010:2.1:1787761878.725184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f861fe00.
00010000:00000010:2.1:1787761878.725186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78be000.
00010000:00000010:2.1:1787761878.725188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05c00.
00010000:00000010:2.1:1787761878.725191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78be200.
00010000:00000010:2.1:1787761878.725192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05800.
00010000:00000010:2.1:1787761878.725194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78be100.
00010000:00000010:2.1:1787761878.725196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05e00.
00010000:00000010:2.1:1787761878.725198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78be400.
00010000:00000010:2.1:1787761878.725199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05400.
00010000:00000010:2.1:1787761878.725201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78be900.
00010000:00000010:2.1:1787761878.725202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05a00.
00010000:00000010:2.1:1787761878.725204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78bed00.
00010000:00000010:2.1:1787761878.725206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05000.
00010000:00000010:2.1:1787761878.725207:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78be500.
00010000:00000010:2.1:1787761878.725209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec04800.
00010000:00000010:2.1:1787761878.725211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78bef00.
00010000:00000010:2.1:1787761878.725212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05200.
00010000:00000010:2.1:1787761878.725214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78bee00.
00010000:00000010:2.1:1787761878.725216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec05600.
00010000:00000010:2.1:1787761878.725218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751d78bec00.
00010000:00000010:2.1:1787761878.725220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751fec04000.
00010000:00000010:2.1:1787761878.725224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8587b00.
00010000:00000010:2.1:1787761878.725227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539a200.
00010000:00000010:2.1:1787761878.725230:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8587800.
00010000:00000010:2.1:1787761878.725232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539aa00.
00010000:00000010:2.1:1787761878.725234:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8587300.
00010000:00000010:2.1:1787761878.725236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ac00.
00010000:00000010:2.1:1787761878.725238:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8587600.
00010000:00000010:2.1:1787761878.725240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539be00.
00010000:00000010:2.1:1787761878.725242:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8587a00.
00010000:00000010:2.1:1787761878.725244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539bc00.
00010000:00000010:2.1:1787761878.725245:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8587700.
00010000:00000010:2.1:1787761878.725247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539a000.
00010000:00000010:2.1:1787761878.725248:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8587000.
00010000:00000010:2.1:1787761878.725250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539b600.
00010000:00000010:2.1:1787761878.725253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2e100.
00010000:00000010:2.1:1787761878.725255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539b200.
00010000:00000010:2.1:1787761878.725257:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2e300.
00010000:00000010:2.1:1787761878.725258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539a600.
00010000:00000010:2.1:1787761878.725260:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2e600.
00010000:00000010:2.1:1787761878.725262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539a800.
00010000:00000010:2.1:1787761878.725264:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2ea00.
00010000:00000010:2.1:1787761878.725265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539b000.
00010000:00000010:2.1:1787761878.725267:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2e700.
00010000:00000010:2.1:1787761878.725269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ba00.
00010000:00000010:2.1:1787761878.725270:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2e000.
00010000:00000010:2.1:1787761878.725272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539ae00.
00010000:00000010:2.1:1787761878.725274:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2e200.
00010000:00000010:2.1:1787761878.725276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539b400.
00010000:00000010:2.1:1787761878.725278:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2e400.
00010000:00000010:2.1:1787761878.725280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539a400.
00010000:00000010:2.1:1787761878.725282:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2e900.
00010000:00000010:2.1:1787761878.725284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c539b800.
00010000:00000010:2.1:1787761878.725286:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2ed00.
00010000:00000010:2.1:1787761878.725289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5399e00.
00010000:00000010:2.1:1787761878.725291:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2e500.
00010000:00000010:2.1:1787761878.725293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5399c00.
00010000:00000010:2.1:1787761878.725295:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2ef00.
00010000:00000010:2.1:1787761878.725298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5398000.
00010000:00000010:2.1:1787761878.729166:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2ee00.
00010000:00000010:2.1:1787761878.729172:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5399600.
00010000:00000010:2.1:1787761878.729174:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2ec00.
00010000:00000010:2.1:1787761878.729176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5399200.
00010000:00000010:2.1:1787761878.729178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2eb00.
00010000:00000010:2.1:1787761878.729180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5398600.
00010000:00000010:2.1:1787761878.729182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751dfb2e800.
00010000:00000010:2.1:1787761878.729183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5398800.
00010000:00000010:2.1:1787761878.729186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780100.
00010000:00000010:2.1:1787761878.729187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5399000.
00010000:00000010:2.1:1787761878.729189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780400.
00010000:00000010:2.1:1787761878.729191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5399a00.
00010000:00000010:2.1:1787761878.729193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780900.
00010000:00000010:2.1:1787761878.729195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5398e00.
00010000:00000010:2.1:1787761878.729197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780d00.
00010000:00000010:2.1:1787761878.729199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5399400.
00010000:00000010:2.1:1787761878.729201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780500.
00010000:00000010:2.1:1787761878.729203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5398400.
00010000:00000010:2.1:1787761878.729204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780f00.
00010000:00000010:2.1:1787761878.729206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5399800.
00010000:00000010:2.1:1787761878.729207:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780e00.
00010000:00000010:2.1:1787761878.729209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5398200.
00010000:00000010:2.1:1787761878.729211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780c00.
00010000:00000010:2.1:1787761878.729212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5398a00.
00010000:00000010:2.1:1787761878.729214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780b00.
00010000:00000010:2.1:1787761878.729215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751c5398c00.
00010000:00000010:2.1:1787761878.729218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9751f8780800.
00010000:00000010:2.1:1787761878.729220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9751f8796e00.
00000800:00000200:1.1:1787761879.101257:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761879.101270:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787761879.102378:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761879.102384:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787761879.102785:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761879.102788:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787761879.103227:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761879.103231:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787761884.221174:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761884.221182:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787761884.222259:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761884.222263:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787761884.222632:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761884.222635:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787761884.223100:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787761884.223103:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787761889.340931:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761889.340938:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787761889.341390:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761889.341391:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787761889.341632:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761889.341634:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787761889.341851:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761889.341854:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787761894.461224:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761894.461832:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761894.462184:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761894.462352:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761899.581081:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761899.581087:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787761899.581781:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761899.581785:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787761899.582107:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761899.582109:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787761899.582388:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761899.582390:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787761904.701270:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761904.701282:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787761904.702618:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761904.702621:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787761904.702855:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761904.702857:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787761904.703164:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761904.703166:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787761909.821364:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761909.821376:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787761909.822418:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761909.822422:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787761909.822827:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761909.822831:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787761909.823252:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761909.823256:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787761914.941405:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761914.941428:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787761914.942403:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761914.942407:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787761914.942835:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761914.942841:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787761914.943271:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761914.943275:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787761920.060961:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761920.060969:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787761920.061483:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761920.061485:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787761920.061742:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761920.061744:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787761920.061952:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761920.061954:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787761925.180901:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761925.180906:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787761925.181381:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761925.181383:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787761925.181631:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761925.181633:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787761925.181859:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761925.181862:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787761930.300850:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761930.300856:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787761930.301402:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761930.301404:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787761930.301633:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761930.301635:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787761930.301823:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787761930.301825:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787761935.421848:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761935.421853:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787761935.422338:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761935.422342:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787761935.422732:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761935.422736:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787761935.423269:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761935.423272:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787761940.541049:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761940.541053:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787761940.541890:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761940.541892:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787761940.542106:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761940.542109:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787761940.542497:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761940.542498:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787761945.660910:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761945.660918:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787761945.661530:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761945.661533:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787761945.661769:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761945.661771:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787761945.662020:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761945.662022:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787761950.780726:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761950.780732:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787761950.781244:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761950.781246:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787761950.781463:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761950.781464:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787761950.781642:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761950.781644:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787761955.900884:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761955.900889:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787761955.901359:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761955.901362:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787761955.901593:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761955.901595:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787761955.901798:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761955.901800:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787761961.020712:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761961.021011:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761961.021200:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761961.021353:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761966.140792:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761966.140798:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787761966.141549:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761966.141552:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787761966.141868:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761966.141871:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787761966.142096:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761966.142098:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787761971.261471:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761971.261483:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787761971.262453:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761971.262457:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787761971.262840:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761971.262843:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787761971.263284:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761971.263287:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787761976.381527:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761976.381537:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787761976.382407:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761976.382411:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787761976.382806:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761976.382809:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787761976.383208:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761976.383212:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787761981.501621:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761981.501631:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787761981.502539:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761981.502545:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787761981.502954:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761981.502958:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787761981.503410:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761981.503413:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:0.1:1787761986.621332:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761986.621339:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787761986.622275:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761986.622278:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787761986.622646:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761986.622649:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787761986.622988:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761986.622992:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787761991.741336:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761991.741347:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787761991.742391:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761991.742395:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787761991.742829:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761991.742832:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787761991.743284:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761991.743289:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787761996.861173:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761996.861177:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787761996.862488:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761996.862490:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787761996.862702:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761996.862704:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787761996.862946:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787761996.862948:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787762001.981658:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762001.981673:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787762001.982664:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762001.982667:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787762001.983215:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762001.983221:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787762001.983604:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762001.983607:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787762007.101309:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762007.101320:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787762007.102351:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762007.102354:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787762007.102783:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762007.102787:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787762007.103215:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762007.103220:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787762012.220956:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762012.220962:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787762012.221516:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762012.221517:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787762012.221738:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762012.221740:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787762012.221943:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762012.221944:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787762017.341088:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762017.341096:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1787762017.342729:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762017.342737:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1787762017.343438:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762017.343441:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1787762017.343919:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762017.343922:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1787762022.461380:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762022.461390:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787762022.462571:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762022.462574:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787762022.463024:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762022.463027:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787762022.463514:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762022.463518:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787762027.581417:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762027.582052:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762027.582413:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762027.582751:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762032.700859:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762032.700867:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787762032.701628:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762032.701631:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787762032.701946:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762032.701949:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787762032.702184:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762032.702186:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787762037.821191:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762037.821201:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787762037.822047:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762037.822050:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787762037.822526:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762037.822529:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787762037.822964:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762037.822970:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787762042.940914:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762042.940922:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787762042.941416:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762042.941418:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787762042.941665:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762042.941668:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787762042.941921:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762042.941923:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787762048.060970:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762048.060977:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787762048.061851:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762048.061854:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787762048.062187:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762048.062189:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787762048.062570:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762048.062574:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787762053.181052:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762053.181054:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787762053.181365:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762053.181367:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787762053.181553:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762053.181555:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787762053.181746:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762053.181747:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787762058.300735:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762058.300740:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787762058.301157:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762058.301160:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787762058.301383:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762058.301386:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787762058.301568:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762058.301570:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787762063.421399:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762063.421417:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787762063.422509:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762063.422511:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787762063.422784:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762063.422786:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787762063.422978:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762063.422980:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787762068.541975:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762068.541980:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762068.542358:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762068.542360:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762068.542599:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762068.542601:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762068.542927:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762068.542929:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762073.661264:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762073.661276:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787762073.662764:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762073.662766:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787762073.662984:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762073.662986:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787762073.663249:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762073.663252:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787762078.781268:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762078.781278:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762078.782328:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762078.782334:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762078.782840:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762078.782844:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762078.783219:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762078.783223:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762083.900646:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762083.900651:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787762083.901026:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762083.901028:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787762083.901293:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762083.901296:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787762083.901572:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762083.901575:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787762089.021509:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762089.021535:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787762089.022643:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762089.022645:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787762089.022863:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762089.022864:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787762089.023179:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762089.023181:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787762094.140793:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762094.141041:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762094.141188:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762094.141294:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762099.261052:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762099.261062:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787762099.261753:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762099.261757:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787762099.262051:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762099.262054:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787762099.262284:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762099.262287:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787762104.380857:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762104.380865:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1787762104.381342:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762104.381346:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1787762104.381554:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762104.381556:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1787762104.381741:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762104.381743:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:1.1:1787762109.501414:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762109.501423:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787762109.502516:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762109.502518:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787762109.502798:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762109.502800:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787762109.503182:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762109.503186:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787762114.621373:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762114.621384:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787762114.622432:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762114.622436:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787762114.622818:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762114.622822:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787762114.623201:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762114.623204:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787762119.741315:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762119.741323:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787762119.742445:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762119.742449:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787762119.742822:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762119.742825:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787762119.743287:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762119.743291:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787762124.860683:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762124.860687:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787762124.861369:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762124.861371:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787762124.861574:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762124.861575:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787762124.861839:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762124.861841:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787762129.981187:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762129.981194:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787762129.982225:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762129.982228:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787762129.982558:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762129.982561:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787762129.982942:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762129.982945:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787762135.101174:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762135.101177:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762135.101556:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762135.101558:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762135.101759:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762135.101761:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762135.101936:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762135.101938:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787762140.221292:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762140.221300:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787762140.222222:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762140.222226:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787762140.222594:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762140.222597:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787762140.222941:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762140.222944:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787762145.340674:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762145.340681:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1787762145.341301:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762145.341303:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1787762145.341503:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762145.341505:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1787762145.341735:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762145.341737:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:0.1:1787762150.461332:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762150.461344:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787762150.462288:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762150.462292:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787762150.462651:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762150.462654:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787762150.463022:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762150.463024:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:0.1:1787762155.581242:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762155.581253:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787762155.582433:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762155.582437:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787762155.582912:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762155.582915:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787762155.583378:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762155.583381:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787762160.701281:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762160.702007:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762160.702482:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762160.702805:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762165.821637:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762165.821648:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787762165.822892:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762165.822898:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787762165.823443:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762165.823449:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787762165.823914:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762165.823918:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787762170.940750:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762170.940754:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787762170.941331:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762170.941334:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787762170.941581:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762170.941583:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787762170.941780:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762170.941782:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787762176.060811:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762176.060818:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787762176.061249:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762176.061251:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787762176.061444:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762176.061446:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787762176.061652:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762176.061654:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787762181.181178:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762181.181193:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787762181.182401:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762181.182404:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787762181.182657:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762181.182660:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787762181.182942:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762181.182944:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787762186.301431:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762186.301444:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787762186.302633:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762186.302637:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787762186.303123:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762186.303127:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787762186.303630:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762186.303634:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787762191.421458:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762191.421473:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787762191.422327:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762191.422331:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787762191.422808:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762191.422812:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787762191.423283:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762191.423288:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787762196.541285:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762196.541295:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787762196.542417:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762196.542421:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787762196.543108:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762196.543114:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787762196.543556:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762196.543560:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787762201.662279:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762201.662286:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762201.662800:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762201.662804:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762201.663269:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762201.663276:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762201.663718:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762201.663722:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762206.781210:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762206.781221:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787762206.782306:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762206.782312:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787762206.782746:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762206.782749:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787762206.783099:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762206.783103:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787762211.900853:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762211.900859:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762211.901262:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762211.901264:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762211.901535:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762211.901552:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762211.901763:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762211.901766:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787762217.020903:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762217.020910:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787762217.021357:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762217.021359:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787762217.021620:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762217.021621:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787762217.021873:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762217.021874:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787762222.141363:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762222.141378:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787762222.142284:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762222.142288:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787762222.142835:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762222.142839:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787762222.143295:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762222.143298:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787762227.260846:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762227.261123:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762227.261287:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762227.261480:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762232.381002:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762232.381011:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787762232.381826:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762232.381831:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787762232.382170:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762232.382175:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787762232.382392:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762232.382394:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787762237.500821:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762237.500828:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787762237.501393:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762237.501395:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787762237.501610:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762237.501611:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787762237.501796:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762237.501798:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787762242.621356:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762242.621370:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787762242.622035:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762242.622039:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787762242.622581:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762242.622585:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787762242.622957:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762242.622963:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787762247.741362:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762247.741371:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787762247.742521:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762247.742523:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787762247.742748:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762247.742750:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787762247.743035:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762247.743038:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787762252.861407:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762252.861423:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787762252.862449:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762252.862452:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787762252.862838:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762252.862841:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787762252.863217:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762252.863220:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787762257.981689:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762257.981706:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787762257.982744:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762257.982748:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787762257.983300:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762257.983304:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787762257.983680:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762257.983684:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787762263.101392:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762263.101401:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787762263.102449:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762263.102451:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787762263.102662:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762263.102664:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787762263.102849:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762263.102851:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1787762268.222249:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762268.222254:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1787762268.222684:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762268.222687:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1787762268.223216:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762268.223219:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1787762268.223556:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762268.223560:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:2.1:1787762273.341546:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762273.341551:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787762273.342529:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762273.342531:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787762273.342764:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762273.342767:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787762273.342973:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762273.342975:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787762278.461434:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762278.461445:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762278.462370:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762278.462375:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762278.462855:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762278.462859:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762278.463408:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762278.463413:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762283.581414:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762283.581424:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787762283.582408:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762283.582412:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787762283.582784:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762283.582788:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787762283.583318:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762283.583324:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787762288.701304:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762288.701314:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787762288.702198:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762288.702202:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787762288.702641:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762288.702645:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787762288.703053:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762288.703059:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787762293.820747:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762293.821082:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762293.821230:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762293.821391:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762298.941357:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762298.941363:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787762298.942978:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762298.942982:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787762298.943619:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762298.943625:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787762298.944442:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762298.944447:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787762304.061012:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762304.061022:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787762304.062558:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762304.062566:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787762304.063068:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762304.063071:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787762304.063551:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762304.063554:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787762309.181322:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762309.181331:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787762309.182479:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762309.182482:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787762309.182936:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762309.182940:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787762309.183414:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762309.183417:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787762314.301426:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762314.301435:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787762314.302291:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762314.302295:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787762314.302654:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762314.302657:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787762314.303044:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762314.303049:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787762319.421260:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762319.421265:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787762319.422454:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762319.422458:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787762319.422934:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762319.422937:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787762319.423424:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762319.423427:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787762324.541462:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762324.541483:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787762324.542081:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762324.542085:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787762324.542669:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762324.542672:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787762324.543069:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762324.543073:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787762329.661227:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762329.661242:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787762329.662563:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762329.662567:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787762329.662806:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762329.662808:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787762329.663039:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762329.663041:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787762334.780770:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762334.780774:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787762334.781255:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762334.781258:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787762334.781628:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762334.781630:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787762334.781936:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762334.781939:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787762339.901255:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762339.901269:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787762339.902636:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762339.902639:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787762339.903183:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762339.903187:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787762339.903608:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762339.903612:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787762345.020761:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762345.020767:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787762345.021216:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762345.021218:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787762345.021412:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762345.021414:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787762345.021645:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762345.021647:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787762350.141388:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762350.141397:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787762350.142350:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762350.142354:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787762350.142730:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762350.142733:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787762350.143051:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762350.143054:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787762355.261271:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762355.261283:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787762355.262000:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762355.262005:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787762355.262400:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762355.262402:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787762355.262770:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762355.262773:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787762360.380712:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762360.381093:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762360.381250:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762360.381435:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762365.501770:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762365.501774:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787762365.502246:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762365.502249:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787762365.502572:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762365.502573:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787762365.502823:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762365.502825:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787762370.620796:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762370.620803:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787762370.621337:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762370.621338:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787762370.621563:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762370.621565:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787762370.621742:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762370.621744:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787762375.741456:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762375.741470:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787762375.742908:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762375.742911:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787762375.743329:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762375.743333:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787762375.743874:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762375.743879:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787762380.861418:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762380.861430:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787762380.862572:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762380.862575:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787762380.862946:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762380.862949:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787762380.863447:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762380.863451:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787762385.981267:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762385.981277:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787762385.982389:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762385.982392:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787762385.982768:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762385.982772:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787762385.983109:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762385.983112:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787762391.101532:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762391.101544:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787762391.102338:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762391.102342:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787762391.102745:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762391.102751:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787762391.103174:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762391.103178:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787762396.221420:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762396.221428:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787762396.223021:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762396.223024:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787762396.223417:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762396.223419:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787762396.223670:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762396.223674:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787762401.341858:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762401.341867:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787762401.342845:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762401.342848:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787762401.343315:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762401.343321:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787762401.343755:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762401.343760:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787762406.461498:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762406.461510:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787762406.462455:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762406.462459:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787762406.462898:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762406.462902:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787762406.463419:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762406.463422:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787762411.581406:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762411.581420:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787762411.582652:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762411.582654:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787762411.582910:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762411.582912:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787762411.583218:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762411.583221:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787762416.701676:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762416.701697:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787762416.702846:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762416.702852:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787762416.703386:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762416.703390:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787762416.703811:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762416.703814:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787762421.821457:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762421.821641:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787762421.822325:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762421.822329:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787762421.822758:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762421.822762:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787762421.823129:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762421.823158:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787762426.941384:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762426.941977:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762426.942354:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762426.942665:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762432.060799:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762432.060807:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787762432.061561:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762432.061564:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787762432.061872:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762432.061875:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787762432.062171:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762432.062174:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787762437.181033:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762437.181040:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787762437.181751:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762437.181755:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787762437.182190:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762437.182193:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787762437.182550:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762437.182553:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787762442.301318:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762442.301330:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787762442.302329:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762442.302333:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787762442.302763:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762442.302766:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787762442.303216:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762442.303219:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787762447.421317:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762447.421326:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787762447.422364:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762447.422368:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787762447.422842:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762447.422846:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787762447.423341:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762447.423345:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787762452.541246:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762452.541252:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787762452.542533:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762452.542544:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787762452.542797:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762452.542799:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787762452.543120:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762452.543122:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787762457.661423:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762457.661431:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787762457.662216:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762457.662219:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787762457.662599:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762457.662604:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787762457.663040:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762457.663044:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787762462.781238:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762462.781249:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787762462.782351:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762462.782354:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787762462.782718:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762462.782721:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787762462.783091:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762462.783094:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787762467.901707:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762467.901720:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787762467.902649:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762467.902651:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787762467.902893:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762467.902895:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787762467.903174:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762467.903177:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787762473.020736:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762473.020740:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787762473.021273:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762473.021275:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787762473.021507:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762473.021511:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787762473.022089:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762473.022093:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787762478.141260:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762478.141267:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787762478.142369:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762478.142373:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787762478.142765:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762478.142768:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787762478.143115:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762478.143118:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787762483.261439:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762483.261452:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787762483.262371:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762483.262375:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787762483.262771:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762483.262775:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787762483.263272:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762483.263275:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787762488.380785:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762488.380791:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1787762488.381207:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762488.381209:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1787762488.381403:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762488.381405:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1787762488.381660:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762488.381662:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1787762493.501356:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762493.501972:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762493.502212:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762493.502549:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762498.621618:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762498.621626:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787762498.622850:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762498.622854:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787762498.623308:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762498.623311:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787762498.623894:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762498.623898:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787762503.741429:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762503.741450:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787762503.742249:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762503.742253:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787762503.742658:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762503.742661:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787762503.743007:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762503.743010:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787762508.861255:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762508.861265:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787762508.862610:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762508.862612:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787762508.862816:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762508.862817:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787762508.863047:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762508.863048:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787762513.981234:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762513.981242:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787762513.982033:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762513.982036:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787762513.982444:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762513.982448:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787762513.982947:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762513.982950:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787762519.100755:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762519.100759:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787762519.101418:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762519.101422:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787762519.101796:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762519.101798:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787762519.101989:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762519.101990:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787762524.221033:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762524.221043:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787762524.221724:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762524.221726:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787762524.221930:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762524.221932:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787762524.222120:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762524.222122:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787762529.341303:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762529.341310:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787762529.342018:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762529.342022:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787762529.342456:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762529.342459:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787762529.342808:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762529.342811:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787762534.461193:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762534.461197:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762534.461546:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762534.461547:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762534.461835:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762534.461837:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762534.462031:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762534.462032:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762539.580777:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762539.580784:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787762539.581250:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762539.581252:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787762539.581470:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762539.581472:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787762539.581685:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762539.581686:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787762544.701381:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762544.701389:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762544.702793:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762544.702797:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762544.703199:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762544.703202:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762544.703603:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762544.703606:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762549.821240:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762549.821251:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787762549.822427:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762549.822430:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787762549.822827:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762549.822831:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787762549.823285:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762549.823289:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787762554.941510:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762554.941514:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787762554.941960:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762554.941964:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787762554.942376:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762554.942379:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787762554.942708:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762554.942711:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787762560.061390:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762560.062093:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762560.062548:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762560.062925:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762565.181727:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762565.181741:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787762565.182690:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762565.182694:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787762565.183181:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762565.183184:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787762565.183547:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762565.183551:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787762570.301522:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762570.301564:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1787762570.302614:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762570.302616:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1787762570.302918:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762570.302921:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1787762570.303297:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762570.303299:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:2.1:1787762575.420743:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762575.420750:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1787762575.421209:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762575.421211:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1787762575.421466:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762575.421468:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1787762575.421654:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762575.421656:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1787762580.540769:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762580.540782:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787762580.541236:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762580.541238:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787762580.541432:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762580.541434:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787762580.541651:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762580.541652:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787762585.661233:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762585.661241:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787762585.661918:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762585.661921:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787762585.662306:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762585.662308:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787762585.662575:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762585.662578:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787762590.780752:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762590.780755:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787762590.781411:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762590.781416:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787762590.781716:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762590.781717:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787762590.781895:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762590.781897:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787762595.901225:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762595.901230:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787762595.902286:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762595.902292:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787762595.902511:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762595.902513:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787762595.902785:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762595.902787:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787762601.021474:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762601.021482:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787762601.022666:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762601.022671:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787762601.023209:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762601.023212:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787762601.023573:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762601.023576:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762606.141313:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762606.141322:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787762606.142515:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762606.142519:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787762606.142983:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762606.142986:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787762606.143467:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762606.143471:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787762611.261249:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762611.261257:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762611.262175:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762611.262178:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762611.262543:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762611.262546:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762611.262912:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762611.262916:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762616.381294:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762616.381324:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787762616.382496:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762616.382500:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787762616.382973:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762616.382976:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787762616.383387:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762616.383392:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787762621.501330:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762621.501345:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787762621.502583:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762621.502589:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787762621.503086:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762621.503089:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787762621.503718:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762621.503741:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787762626.621635:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762626.622007:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762626.622285:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762626.622565:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762631.741828:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762631.741846:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787762631.742730:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762631.742735:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787762631.743243:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762631.743247:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787762631.743763:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762631.743769:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787762636.861531:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762636.861543:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787762636.862501:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762636.862505:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787762636.862983:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762636.862987:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787762636.863393:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762636.863396:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787762641.981385:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762641.981408:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787762641.982301:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762641.982304:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787762641.982703:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762641.982706:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787762641.983051:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762641.983054:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787762647.100828:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762647.100837:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787762647.101414:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762647.101416:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787762647.101667:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762647.101668:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787762647.101903:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762647.101905:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787762652.221333:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762652.221344:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787762652.222491:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762652.222495:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787762652.222962:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762652.222966:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787762652.223410:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762652.223414:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787762657.341358:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762657.341368:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787762657.342422:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762657.342425:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787762657.342826:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762657.342829:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787762657.343202:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762657.343205:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787762662.461263:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762662.461273:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787762662.462548:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762662.462554:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787762662.463176:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762662.463178:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787762662.463371:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762662.463373:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787762667.581229:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762667.581232:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787762667.581611:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762667.581613:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787762667.581835:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762667.581837:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787762667.582010:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762667.582012:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787762672.700782:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762672.700786:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787762672.701250:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762672.701252:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787762672.701472:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762672.701474:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787762672.701690:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762672.701692:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787762677.821431:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762677.821443:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787762677.822231:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762677.822237:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787762677.822681:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762677.822684:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787762677.823026:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762677.823029:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787762682.941229:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762682.941242:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787762682.942219:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762682.942223:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787762682.942631:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762682.942634:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787762682.942982:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762682.942986:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787762688.061283:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762688.061296:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787762688.062372:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762688.062376:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787762688.062748:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762688.062752:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787762688.063088:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762688.063092:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787762693.181215:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762693.182009:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762693.182417:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762693.182725:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762698.301069:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762698.301078:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:2.1:1787762698.301879:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762698.301882:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:2.1:1787762698.302182:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762698.302184:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:2.1:1787762698.302445:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762698.302449:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:2.1:1787762703.420744:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762703.420751:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:2.1:1787762703.421236:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762703.421237:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:2.1:1787762703.421437:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762703.421439:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:2.1:1787762703.421612:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762703.421614:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:2.1:1787762708.540846:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762708.540851:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1787762708.541301:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762708.541303:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1787762708.541500:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762708.541502:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:2.1:1787762708.541704:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762708.541706:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787762713.660766:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762713.660772:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787762713.661207:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762713.661209:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787762713.661403:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762713.661405:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787762713.661632:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762713.661634:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787762718.781236:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762718.781242:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787762718.782072:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762718.782077:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787762718.782838:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762718.782840:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787762718.783032:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762718.783034:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787762723.901385:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762723.901398:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787762723.902392:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762723.902396:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787762723.902782:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762723.902785:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787762723.903236:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762723.903240:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787762729.020718:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762729.020723:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787762729.021248:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762729.021249:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787762729.021441:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762729.021443:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787762729.021641:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762729.021643:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787762734.141864:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762734.141875:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762734.142636:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762734.142639:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762734.143027:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762734.143030:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762734.143411:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762734.143414:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787762739.261344:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762739.261356:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787762739.262209:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762739.262212:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787762739.262621:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762739.262627:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787762739.263023:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762739.263027:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787762744.381034:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762744.381038:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:2.1:1787762744.382087:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762744.382092:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:2.1:1787762744.382694:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762744.382696:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:2.1:1787762744.382889:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762744.382891:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:2.1:1787762749.501245:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762749.501256:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1787762749.502534:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762749.502539:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1787762749.502972:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762749.502976:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1787762749.503490:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762749.503494:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:2.1:1787762754.621279:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762754.621289:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787762754.622272:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762754.622276:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787762754.622707:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762754.622710:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787762754.623129:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762754.623158:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:2.1:1787762759.741287:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762759.741990:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762759.742691:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762759.742858:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762764.862323:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762764.862330:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787762764.862943:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762764.862949:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787762764.863456:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762764.863459:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787762764.863910:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762764.863914:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787762769.981479:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762769.981490:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787762769.982402:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762769.982405:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787762769.982824:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762769.982830:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787762769.983344:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762769.983349:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787762775.101314:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762775.101326:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787762775.102367:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762775.102370:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787762775.102769:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762775.102772:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787762775.103122:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762775.103125:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787762780.220814:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762780.220817:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787762780.221306:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762780.221308:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787762780.221514:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762780.221516:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787762780.221708:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762780.221709:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787762785.340835:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762785.340841:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787762785.341475:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762785.341477:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787762785.341696:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762785.341697:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787762785.341934:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762785.341935:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787762790.460756:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762790.460761:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787762790.461272:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762790.461274:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787762790.461468:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762790.461470:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787762790.461695:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762790.461697:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787762795.580724:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762795.580730:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787762795.581292:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762795.581293:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787762795.581487:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762795.581489:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787762795.581719:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762795.581720:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787762800.700877:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762800.700883:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787762800.701541:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762800.701543:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787762800.701795:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762800.701796:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787762800.702003:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762800.702004:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787762805.820996:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762805.821003:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787762805.821590:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762805.821592:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787762805.821841:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762805.821843:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787762805.822037:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762805.822039:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787762810.940937:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762810.940943:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787762810.941568:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762810.941570:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787762810.941777:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762810.941778:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787762810.941952:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762810.941954:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787762816.060767:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762816.060771:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787762816.061436:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762816.061438:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787762816.061682:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762816.061684:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787762816.061920:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762816.061922:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787762821.181341:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762821.181374:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787762821.182275:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762821.182279:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787762821.182694:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762821.182697:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787762821.183039:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762821.183042:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787762826.301293:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762826.302125:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762826.302588:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762826.302906:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762831.421498:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762831.421513:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787762831.422643:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762831.422647:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787762831.423211:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762831.423217:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787762831.423629:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762831.423632:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787762836.541099:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762836.541108:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787762836.541745:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762836.541747:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787762836.541962:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762836.541963:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787762836.542224:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762836.542226:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:0.1:1787762841.661431:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762841.661445:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787762841.662544:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762841.662548:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787762841.662922:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762841.662926:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787762841.663379:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762841.663383:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:0.1:1787762846.781350:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762846.781362:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787762846.782391:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762846.782395:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787762846.782765:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762846.782768:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787762846.783108:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762846.783110:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787762851.901279:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762851.901285:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787762851.902558:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762851.902569:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787762851.903013:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762851.903015:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787762851.903319:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762851.903321:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:0.1:1787762857.020934:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762857.020942:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787762857.021786:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762857.021788:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787762857.022090:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762857.022092:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787762857.022450:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762857.022453:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:0.1:1787762862.140817:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762862.140824:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787762862.141320:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762862.141321:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787762862.141535:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762862.141537:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787762862.141723:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787762862.141725:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787762867.261272:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762867.261280:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:2.1:1787762867.261632:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762867.261635:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:2.1:1787762867.261953:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762867.261955:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:2.1:1787762867.262177:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787762867.262179:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:1.1:1787762872.381486:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762872.381494:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787762872.382611:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762872.382614:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787762872.382984:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762872.382987:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787762872.383501:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762872.383505:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787762877.500796:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762877.500801:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787762877.501355:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762877.501357:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787762877.501564:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762877.501568:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787762877.501790:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762877.501793:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787762882.621055:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762882.621062:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787762882.621784:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762882.621788:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787762882.622221:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762882.622225:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787762882.622571:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762882.622574:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787762887.741121:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762887.741130:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787762887.741630:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762887.741632:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787762887.741978:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762887.741981:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787762887.742360:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762887.742364:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787762892.860813:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762892.861090:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762892.861210:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762892.861372:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762897.980807:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762897.980815:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787762897.981574:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762897.981577:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787762897.981913:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762897.981916:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787762897.982247:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762897.982250:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:0.0:1787762903.100297:0:304298:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787762903.100301:0:304298:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787762903.100303:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762903.100305:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762903.100311:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762903.100314:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762903.100317:0:304298:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787762903.100320:0:304298:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762903.100322:0:304298:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762903.100324:0:304298:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762903.100327:0:304298:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762903.100329:0:304298:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762903.100330:0:304298:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787762903.100332:0:304298:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762903.100340:0:304298:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000020:00000040:0.0:1787762903.100342:0:304298:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787762903.100345:0:304298:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762903.100346:0:304298:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762903.100348:0:304298:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762903.100350:0:304298:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762903.100353:0:304298:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762903.100359:0:304298:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787762903.100365:0:304298:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762903.100371:0:304298:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832571264/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762903.100379:0:304298:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762903.100383:0:304298:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762903.100386:0:304298:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832571264/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762903.100402:0:304298:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787762903.100404:0:304298:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762903.100412:0:304298:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787762903.100414:0:304298:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762903.100415:0:304298:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762903.100418:0:304298:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762903.100420:0:304298:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20617
00000400:00000200:0.0:1787762903.100423:0:304298:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762903.100426:0:304298:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762903.100428:0:304298:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762903.100431:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762903.100433:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762903.100437:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762903.100442:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762903.100446:0:304298:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762903.100447:0:304298:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787762903.100448:0:304298:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762903.100451:0:304298:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787762903.100453:0:304298:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787762903.100457:0:304298:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787762903.100464:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762903.100468:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762903.100473:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762903.100476:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762903.100479:0:304298:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787762903.100484:0:304298:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762903.100490:0:304298:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762903.100494:0:304298:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762903.100501:0:304298:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762903.100506:0:304298:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762903.100507:0:304298:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787762903.100513:0:304298:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787762903.100518:0:304298:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762903.100520:0:304298:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762903.100528:0:304298:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787762903.100533:0:304298:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787762903.100537:0:304298:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762903.100539:0:304298:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762903.100544:0:304298:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762903.100545:0:304298:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762903.100547:0:304298:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762903.100552:0:304298:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787762903.100556:0:304298:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762903.100566:0:304298:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832571392/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762903.100576:0:304298:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762903.100580:0:304298:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762903.100587:0:304298:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832571392/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762903.100605:0:304298:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787762903.100609:0:304298:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762903.100613:0:304298:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787762903.100616:0:304298:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762903.100617:0:304298:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762903.100619:0:304298:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762903.100624:0:304298:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20617
00000400:00000200:0.0:1787762903.100655:0:304298:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762903.100664:0:304298:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762903.100667:0:304298:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762903.100673:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762903.100677:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762903.100682:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762903.100684:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762903.100688:0:304298:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762903.100690:0:304298:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787762903.100692:0:304298:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762903.100694:0:304298:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787762903.100698:0:304298:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787762903.100706:0:304298:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:1.1:1787762903.100706:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787762903.100713:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:1.1:1787762903.100714:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:0.0:1787762903.100715:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762903.100720:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762903.100724:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762903.100728:0:304298:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787762903.100733:0:304298:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762903.100736:0:304298:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762903.100741:0:304298:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762903.100745:0:304298:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762903.100751:0:304298:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762903.100753:0:304298:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787762903.100756:0:304298:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787762903.100759:0:304298:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762903.100762:0:304298:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762903.100765:0:304298:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
00000020:00000040:0.0:1787762903.100770:0:304298:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787762903.100775:0:304298:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762903.100777:0:304298:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762903.100781:0:304298:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762903.100796:0:304298:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762903.100797:0:304298:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762903.100802:0:304298:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787762903.100807:0:304298:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762903.100815:0:304298:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832571520/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762903.100826:0:304298:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762903.100830:0:304298:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762903.100839:0:304298:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832571520/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762903.100848:0:304298:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762903.100852:0:304298:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787762903.100857:0:304298:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787762903.100860:0:304298:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762903.100864:0:304298:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787762903.100865:0:304298:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762903.100870:0:304298:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762903.100874:0:304298:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762903.100879:0:304298:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20617
00000400:00000200:0.0:1787762903.100886:0:304298:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762903.100895:0:304298:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762903.100902:0:304298:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762903.100910:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762903.100916:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762903.100922:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762903.100925:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762903.100929:0:304298:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762903.100932:0:304298:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787762903.100935:0:304298:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762903.100939:0:304298:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787762903.100944:0:304298:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787762903.100946:0:304298:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787762903.100951:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762903.100956:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762903.100960:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762903.100964:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762903.100970:0:304298:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787762903.100975:0:304298:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762903.100980:0:304298:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762903.100984:0:304298:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762903.100988:0:304298:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762903.100993:0:304298:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762903.100996:0:304298:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787762903.101000:0:304298:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787762903.101006:0:304298:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762903.101010:0:304298:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762903.101017:0:304298:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787762903.101020:0:304298:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787762903.101024:0:304298:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762903.101026:0:304298:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762903.101031:0:304298:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762903.101033:0:304298:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762903.101035:0:304298:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762903.101040:0:304298:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787762903.101046:0:304298:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762903.101056:0:304298:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832571648/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762903.101068:0:304298:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762903.101073:0:304298:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762903.101081:0:304298:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832571648/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762903.101105:0:304298:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762903.101110:0:304298:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20627)
00000800:00000200:1.1:1787762903.101265:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762903.101266:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787762903.101459:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762903.101461:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787762903.101637:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762903.101638:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:0.0:1787762905.788276:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787762905.788282:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787762905.788289:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787762908.220335:0:304298:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762908.220349:0:304298:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787762908.220380:0:304298:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762908.220385:0:304298:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762908.220390:0:304298:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762908.220398:0:304298:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 20622
00000400:00000200:0.0:1787762908.220405:0:304298:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762908.220411:0:304298:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762908.220414:0:304298:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762908.220417:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762908.220420:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762908.220422:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762908.220424:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762908.220428:0:304298:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762908.220429:0:304298:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787762908.220432:0:304298:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762908.220434:0:304298:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787762908.220454:0:304298:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787762908.220458:0:304298:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787762908.220464:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762908.220468:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762908.220472:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762908.220475:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762908.220480:0:304298:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787762908.220485:0:304298:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762908.220490:0:304298:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762908.220494:0:304298:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762908.220498:0:304298:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762908.220502:0:304298:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762908.220505:0:304298:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787762908.220509:0:304298:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762908.220519:0:304298:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787762908.220523:0:304298:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787762908.220530:0:304298:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762908.220532:0:304298:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762908.220536:0:304298:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762908.220540:0:304298:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762908.220544:0:304298:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762908.220554:0:304298:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787762908.220580:0:304298:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762908.220590:0:304298:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832571776/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762908.220598:0:304298:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762908.220602:0:304298:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762908.220607:0:304298:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832571776/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762908.220623:0:304298:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787762908.220626:0:304298:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762908.220650:0:304298:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787762908.220653:0:304298:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762908.220657:0:304298:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762908.220661:0:304298:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762908.220667:0:304298:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20622
00000400:00000200:0.0:1787762908.220674:0:304298:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762908.220679:0:304298:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762908.220682:0:304298:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762908.220688:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762908.220708:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762908.220719:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762908.220729:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762908.220739:0:304298:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762908.220744:0:304298:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787762908.220788:0:304298:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762908.220792:0:304298:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787762908.220798:0:304298:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787762908.220799:0:304298:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787762908.220804:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762908.220808:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762908.220816:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762908.220818:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762908.220825:0:304298:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787762908.220830:0:304298:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762908.220832:0:304298:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762908.220860:0:304298:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762908.220869:0:304298:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762908.220876:0:304298:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762908.220884:0:304298:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787762908.220894:0:304298:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787762908.220903:0:304298:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762908.220906:0:304298:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762908.220913:0:304298:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787762908.220921:0:304298:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787762908.220930:0:304298:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762908.220940:0:304298:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762908.220950:0:304298:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762908.220973:0:304298:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762908.220978:0:304298:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762908.220985:0:304298:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787762908.220993:0:304298:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762908.221004:0:304298:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832571904/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762908.221018:0:304298:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762908.221022:0:304298:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762908.221032:0:304298:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832571904/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762908.221063:0:304298:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787762908.221067:0:304298:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762908.221072:0:304298:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787762908.221075:0:304298:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762908.221079:0:304298:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762908.221086:0:304298:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762908.221091:0:304298:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20622
00000400:00000200:0.0:1787762908.221096:0:304298:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762908.221102:0:304298:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762908.221105:0:304298:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762908.221110:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762908.221113:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762908.221118:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762908.221121:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:1.1:1787762908.221122:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787762908.221124:0:304298:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762908.221127:0:304298:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000800:00000200:1.1:1787762908.221127:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:0.0:1787762908.221131:0:304298:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762908.221134:0:304298:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787762908.221139:0:304298:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787762908.221144:0:304298:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787762908.221168:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762908.221173:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762908.221179:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762908.221181:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762908.221184:0:304298:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787762908.221187:0:304298:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762908.221191:0:304298:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762908.221193:0:304298:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762908.221197:0:304298:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762908.221200:0:304298:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762908.221206:0:304298:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787762908.221210:0:304298:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787762908.221216:0:304298:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762908.221221:0:304298:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762908.221229:0:304298:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787762908.221234:0:304298:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787762908.221258:0:304298:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762908.221263:0:304298:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762908.221272:0:304298:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762908.221274:0:304298:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762908.221278:0:304298:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762908.221291:0:304298:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787762908.221303:0:304298:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762908.221316:0:304298:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832572032/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762908.221342:0:304298:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762908.221350:0:304298:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762908.221362:0:304298:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832572032/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762908.221404:0:304298:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762908.221408:0:304298:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787762908.221413:0:304298:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787762908.221417:0:304298:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762908.221423:0:304298:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787762908.221427:0:304298:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762908.221429:0:304298:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762908.221432:0:304298:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762908.221460:0:304298:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20622
00000400:00000200:0.0:1787762908.221476:0:304298:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762908.221488:0:304298:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762908.221499:0:304298:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762908.221511:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762908.221517:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762908.221526:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762908.221535:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762908.221539:0:304298:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762908.221544:0:304298:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787762908.221565:0:304298:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762908.221568:0:304298:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787762908.221570:0:304298:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787762908.221574:0:304298:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787762908.221580:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762908.221585:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762908.221589:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762908.221594:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762908.221597:0:304298:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787762908.221603:0:304298:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762908.221605:0:304298:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762908.221608:0:304298:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762908.221610:0:304298:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762908.221631:0:304298:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762908.221638:0:304298:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787762908.221644:0:304298:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787762908.221653:0:304298:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762908.221656:0:304298:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762908.221669:0:304298:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000020:00000040:0.0:1787762908.221676:0:304298:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787762908.221684:0:304298:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762908.221691:0:304298:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762908.221700:0:304298:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762908.221705:0:304298:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762908.221711:0:304298:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762908.221722:0:304298:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787762908.221730:0:304298:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762908.221746:0:304298:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832572160/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762908.221807:0:304298:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762908.221812:0:304298:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762908.221819:0:304298:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832572160/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762908.221834:0:304298:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762908.221860:0:304298:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20632)
00000800:00000200:1.1:1787762908.222042:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762908.222044:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787762908.222300:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762908.222303:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787762908.222617:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762908.222637:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:0.0:1787762908.860397:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787762908.860404:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787762908.860408:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787762911.932393:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787762911.932399:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787762911.932404:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787762913.340261:0:304298:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762913.340277:0:304298:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787762913.340282:0:304298:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762913.340289:0:304298:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762913.340297:0:304298:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762913.340307:0:304298:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 20627
00000400:00000200:0.0:1787762913.340320:0:304298:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762913.340329:0:304298:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762913.340333:0:304298:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762913.340337:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762913.340340:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762913.340346:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762913.340351:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762913.340356:0:304298:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762913.340358:0:304298:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787762913.340363:0:304298:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762913.340367:0:304298:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787762913.340371:0:304298:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787762913.340381:0:304298:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787762913.340390:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762913.340395:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762913.340408:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762913.340426:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762913.340438:0:304298:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787762913.340457:0:304298:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762913.340465:0:304298:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762913.340470:0:304298:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762913.340476:0:304298:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762913.340480:0:304298:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762913.340483:0:304298:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787762913.340486:0:304298:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762913.340498:0:304298:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
00000020:00000040:0.0:1787762913.340503:0:304298:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787762913.340508:0:304298:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762913.340511:0:304298:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762913.340515:0:304298:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762913.340519:0:304298:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762913.340525:0:304298:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762913.340534:0:304298:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787762913.340542:0:304298:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762913.340553:0:304298:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832572288/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762913.340562:0:304298:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762913.340574:0:304298:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762913.340580:0:304298:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832572288/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762913.340604:0:304298:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787762913.340608:0:304298:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762913.340612:0:304298:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787762913.340615:0:304298:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762913.340617:0:304298:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762913.340619:0:304298:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762913.340622:0:304298:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20627
00000400:00000200:0.0:1787762913.340626:0:304298:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762913.340629:0:304298:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762913.340631:0:304298:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762913.340633:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762913.340635:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762913.340637:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762913.340638:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762913.340641:0:304298:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762913.340642:0:304298:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787762913.340644:0:304298:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762913.340646:0:304298:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787762913.340648:0:304298:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787762913.340649:0:304298:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787762913.340651:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762913.340653:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762913.340655:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762913.340659:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762913.340662:0:304298:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787762913.340664:0:304298:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762913.340666:0:304298:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762913.340668:0:304298:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762913.340675:0:304298:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762913.340677:0:304298:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762913.340678:0:304298:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787762913.340684:0:304298:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787762913.340689:0:304298:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762913.340693:0:304298:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762913.340703:0:304298:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000020:00000040:0.0:1787762913.340709:0:304298:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787762913.340729:0:304298:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762913.340730:0:304298:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762913.340735:0:304298:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762913.340741:0:304298:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762913.340746:0:304298:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762913.340758:0:304298:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787762913.340766:0:304298:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762913.340786:0:304298:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832572416/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762913.340805:0:304298:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762913.340815:0:304298:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762913.340821:0:304298:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832572416/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762913.340833:0:304298:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787762913.340839:0:304298:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762913.340844:0:304298:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787762913.340847:0:304298:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762913.340855:0:304298:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762913.340859:0:304298:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762913.340865:0:304298:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20627
00000400:00000200:0.0:1787762913.340870:0:304298:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762913.340881:0:304298:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762913.340885:0:304298:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762913.340895:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762913.340906:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762913.340911:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762913.340917:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762913.340919:0:304298:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762913.340920:0:304298:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787762913.340923:0:304298:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762913.340929:0:304298:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787762913.340933:0:304298:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787762913.340948:0:304298:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787762913.340962:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762913.340964:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762913.340965:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762913.340969:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762913.340978:0:304298:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787762913.340995:0:304298:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762913.341005:0:304298:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762913.341013:0:304298:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762913.341018:0:304298:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762913.341023:0:304298:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762913.341025:0:304298:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787762913.341027:0:304298:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787762913.341043:0:304298:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762913.341045:0:304298:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762913.341049:0:304298:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787762913.341055:0:304298:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787762913.341062:0:304298:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762913.341066:0:304298:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762913.341069:0:304298:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762913.341075:0:304298:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762913.341079:0:304298:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762913.341090:0:304298:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787762913.341101:0:304298:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762913.341117:0:304298:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832572544/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762913.341205:0:304298:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762913.341208:0:304298:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762913.341227:0:304298:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832572544/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762913.341254:0:304298:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762913.341267:0:304298:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787762913.341274:0:304298:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787762913.341284:0:304298:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762913.341290:0:304298:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787762913.341291:0:304298:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762913.341293:0:304298:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762913.341295:0:304298:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762913.341304:0:304298:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20627
00000400:00000200:0.0:1787762913.341382:0:304298:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762913.341391:0:304298:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762913.341397:0:304298:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762913.341407:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762913.341413:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762913.341423:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762913.341434:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762913.341444:0:304298:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762913.341451:0:304298:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787762913.341458:0:304298:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762913.341463:0:304298:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787762913.341469:0:304298:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787762913.341482:0:304298:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787762913.341497:0:304298:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762913.341511:0:304298:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762913.341519:0:304298:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000200:1.1:1787762913.341523:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787762913.341525:0:304298:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762913.341537:0:304298:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000800:00000200:1.1:1787762913.341538:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000100:00080000:0.0:1787762913.341553:0:304298:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762913.341565:0:304298:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762913.341579:0:304298:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762913.341590:0:304298:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762913.341595:0:304298:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762913.341608:0:304298:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787762913.341616:0:304298:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787762913.341631:0:304298:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762913.341637:0:304298:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762913.341654:0:304298:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000020:00000040:0.0:1787762913.341660:0:304298:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787762913.341668:0:304298:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762913.341670:0:304298:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762913.341674:0:304298:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762913.341680:0:304298:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762913.341685:0:304298:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762913.341698:0:304298:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787762913.341707:0:304298:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762913.341731:0:304298:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832572672/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762913.341751:0:304298:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762913.341761:0:304298:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762913.341780:0:304298:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832572672/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762913.341812:0:304298:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762913.341820:0:304298:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20638)
00000800:00000200:1.1:1787762913.342315:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762913.342318:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787762913.342695:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762913.342698:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787762913.343077:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762913.343080:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000100:00000001:0.0:1787762915.004326:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762915.004335:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762915.004338:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762915.004344:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762915.004350:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762915.004358:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762915.004363:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762915.004368:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762915.004374:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762915.388366:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762915.388377:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762915.388380:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762915.388387:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762915.388395:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762915.388402:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762915.388407:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762915.388413:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762915.388418:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762916.028306:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762916.028314:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762916.028319:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762916.028326:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762916.028335:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762916.028343:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762916.028347:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762916.028351:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762916.028355:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787762916.028447:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787762916.028451:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787762916.028458:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787762916.412252:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762916.412264:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762916.412269:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762916.412277:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762916.412283:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762916.412289:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762916.412293:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762916.412297:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762916.412300:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762917.436274:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762917.436283:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762917.436286:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762917.436294:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762917.436301:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762917.436307:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762917.436311:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762917.436314:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762917.436318:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787762918.460297:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762918.460309:0:259241:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787762918.460313:0:259241:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762918.460321:0:259241:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762918.460333:0:259241:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762918.460345:0:259241:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 20633
00000400:00000200:0.0:1787762918.460372:0:259241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762918.460385:0:259241:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762918.460391:0:259241:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762918.460397:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762918.460403:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762918.460409:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762918.460413:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762918.460418:0:259241:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762918.460420:0:259241:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787762918.460423:0:259241:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762918.460427:0:259241:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787762918.460430:0:259241:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787762918.460433:0:259241:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787762918.460437:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762918.460440:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762918.460443:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762918.460446:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762918.460450:0:259241:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787762918.460454:0:259241:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762918.460458:0:259241:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762918.460461:0:259241:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762918.460466:0:259241:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762918.460470:0:259241:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762918.460472:0:259241:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787762918.460475:0:259241:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762918.460487:0:259241:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787762918.460491:0:259241:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787762918.460496:0:259241:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762918.460498:0:259241:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762918.460502:0:259241:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762918.460506:0:259241:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762918.460511:0:259241:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762918.460540:0:259241:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787762918.460549:0:259241:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762918.460561:0:259241:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832572800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762918.460575:0:259241:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762918.460580:0:259241:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762918.460587:0:259241:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832572800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762918.460613:0:259241:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787762918.460617:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762918.460622:0:259241:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787762918.460624:0:259241:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762918.460626:0:259241:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762918.460630:0:259241:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762918.460633:0:259241:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20633
00000400:00000200:0.0:1787762918.460639:0:259241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762918.460643:0:259241:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762918.460646:0:259241:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762918.460651:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762918.460655:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762918.460660:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762918.460663:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762918.460666:0:259241:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762918.460667:0:259241:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787762918.460669:0:259241:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762918.460672:0:259241:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787762918.460676:0:259241:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787762918.460702:0:259241:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787762918.460706:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762918.460713:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762918.460716:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762918.460724:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762918.460730:0:259241:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787762918.460739:0:259241:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762918.460743:0:259241:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762918.460750:0:259241:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762918.460762:0:259241:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762918.460768:0:259241:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762918.460774:0:259241:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787762918.460782:0:259241:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787762918.460789:0:259241:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762918.460801:0:259241:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762918.460815:0:259241:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787762918.460821:0:259241:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787762918.460834:0:259241:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762918.460839:0:259241:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762918.460845:0:259241:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762918.460852:0:259241:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762918.460857:0:259241:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762918.460865:0:259241:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787762918.460873:0:259241:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762918.460889:0:259241:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832572928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762918.460913:0:259241:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762918.460920:0:259241:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762918.460936:0:259241:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832572928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762918.460967:0:259241:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787762918.460973:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762918.460991:0:259241:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787762918.460997:0:259241:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762918.461003:0:259241:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762918.461012:0:259241:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762918.461025:0:259241:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20633
00000400:00000200:0.0:1787762918.461036:0:259241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762918.461044:0:259241:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762918.461050:0:259241:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762918.461060:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762918.461067:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762918.461071:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762918.461074:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762918.461084:0:259241:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762918.461090:0:259241:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787762918.461095:0:259241:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762918.461098:0:259241:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787762918.461104:0:259241:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787762918.461113:0:259241:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787762918.461122:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762918.461176:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762918.461187:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762918.461195:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762918.461205:0:259241:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787762918.461219:0:259241:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762918.461229:0:259241:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762918.461234:0:259241:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762918.461241:0:259241:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762918.461247:0:259241:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762918.461255:0:259241:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787762918.461262:0:259241:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787762918.461273:0:259241:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762918.461279:0:259241:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762918.461287:0:259241:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787762918.461294:0:259241:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787762918.461304:0:259241:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762918.461306:0:259241:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762918.461314:0:259241:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762918.461319:0:259241:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762918.461326:0:259241:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762918.461340:0:259241:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000800:00000200:1.1:1787762918.461341:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787762918.461349:0:259241:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787762918.461351:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000100:00100000:0.0:1787762918.461366:0:259241:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832573056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762918.461388:0:259241:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762918.461397:0:259241:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762918.461413:0:259241:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832573056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762918.461433:0:259241:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762918.461441:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787762918.461451:0:259241:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787762918.461458:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762918.461464:0:259241:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787762918.461469:0:259241:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762918.461473:0:259241:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762918.461481:0:259241:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762918.461491:0:259241:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20633
00000400:00000200:0.0:1787762918.461498:0:259241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762918.461502:0:259241:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762918.461505:0:259241:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762918.461508:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762918.461510:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762918.461516:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762918.461523:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762918.461526:0:259241:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762918.461529:0:259241:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787762918.461537:0:259241:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762918.461541:0:259241:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787762918.461547:0:259241:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787762918.461559:0:259241:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787762918.461573:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762918.461582:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762918.461590:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762918.461599:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762918.461607:0:259241:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787762918.461619:0:259241:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762918.461625:0:259241:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762918.461630:0:259241:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762918.461637:0:259241:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762918.461647:0:259241:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762918.461657:0:259241:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787762918.461662:0:259241:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787762918.461669:0:259241:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762918.461671:0:259241:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762918.461699:0:259241:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787762918.461705:0:259241:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787762918.461711:0:259241:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762918.461712:0:259241:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762918.461716:0:259241:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762918.461719:0:259241:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762918.461723:0:259241:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762918.461736:0:259241:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787762918.461743:0:259241:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762918.461759:0:259241:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832573184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762918.461780:0:259241:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762918.461788:0:259241:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762918.461798:0:259241:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832573184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762918.461821:0:259241:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762918.461833:0:259241:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20643)
00000800:00000200:1.1:1787762918.462245:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762918.462249:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787762918.462675:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762918.462701:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787762918.463060:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762918.463063:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000100:00000001:0.0:1787762922.172335:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762922.172346:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762922.172349:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762922.172356:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762922.172362:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762922.172366:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762922.172370:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762922.172373:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762922.172377:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762922.556295:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762922.556304:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762922.556307:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762922.556314:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762922.556319:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762922.556323:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762922.556326:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762922.556329:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762922.556335:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762923.196254:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762923.196268:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762923.196272:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762923.196282:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762923.196289:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762923.196294:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762923.196298:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762923.196301:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762923.196307:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762923.580222:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762923.580231:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762923.580233:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762923.580240:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762923.580245:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762923.580250:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762923.580253:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762923.580257:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762923.580261:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787762923.581172:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762923.581177:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787762923.582496:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762923.582498:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787762923.582748:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762923.582750:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787762923.583040:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762923.583042:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:0.0:1787762924.220222:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762924.220240:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762924.220244:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762924.220250:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762924.220256:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762924.220260:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762924.220262:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762924.220265:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762924.220268:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762924.604214:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762924.604223:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762924.604225:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762924.604232:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762924.604237:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762924.604241:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762924.604244:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762924.604247:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762924.604250:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762925.244205:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762925.244215:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762925.244218:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762925.244227:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762925.244233:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762925.244238:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762925.244242:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762925.244245:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762925.244248:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762925.628237:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762925.628248:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762925.628251:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762925.628258:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762925.628263:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762925.628268:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762925.628272:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762925.628276:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762925.628280:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762926.268235:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762926.268243:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762926.268246:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762926.268252:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762926.268260:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762926.268263:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762926.268267:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762926.268270:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762926.268284:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762926.652201:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762926.652208:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762926.652210:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762926.652215:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762926.652219:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762926.652223:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762926.652226:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762926.652229:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762926.652231:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762927.292195:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762927.292201:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762927.292205:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762927.292211:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762927.292216:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762927.292220:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762927.292223:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762927.292226:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762927.292231:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762927.676207:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762927.676214:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762927.676216:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762927.676220:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762927.676223:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762927.676226:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762927.676229:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762927.676231:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762927.676233:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762928.316218:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762928.316226:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762928.316229:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762928.316234:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762928.316238:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762928.316243:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762928.316248:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762928.316252:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762928.316256:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762928.700208:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762928.700216:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762928.700219:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762928.700223:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762928.700227:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762928.700229:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762928.700232:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762928.700235:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762928.700241:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787762928.700761:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762928.700767:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787762928.701255:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762928.701257:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787762928.701445:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762928.701447:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787762928.701688:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762928.701690:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:0.0:1787762929.340204:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762929.340210:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762929.340214:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762929.340220:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762929.340225:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762929.340230:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762929.340237:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762929.340240:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762929.340244:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762930.364268:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762930.364281:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762930.364284:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762930.364294:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762930.364301:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762930.364308:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762930.364312:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762930.364319:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762930.364323:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762931.388221:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762931.388235:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762931.388239:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762931.388251:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762931.388261:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762931.388269:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762931.388274:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762931.388277:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762931.388283:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762932.412260:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762932.412274:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762932.412280:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762932.412291:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762932.412302:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762932.412313:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762932.412323:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762932.412333:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762932.412346:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762933.436213:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762933.436224:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762933.436226:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762933.436233:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762933.436238:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762933.436244:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762933.436248:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762933.436253:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762933.436259:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762933.820382:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762933.820395:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762933.820403:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762933.820415:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762933.820426:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762933.820436:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762933.820448:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762933.820458:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762933.820469:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787762933.821657:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762933.821666:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762933.822805:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762933.822807:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762933.823018:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762933.823020:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787762933.823291:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762933.823293:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:0.0:1787762934.460265:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762934.460275:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762934.460278:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762934.460287:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762934.460293:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762934.460298:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762934.460301:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762934.460305:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762934.460310:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762934.844304:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762934.844314:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762934.844316:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762934.844323:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762934.844328:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762934.844333:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762934.844352:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762934.844357:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762934.844363:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762935.484280:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762935.484291:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762935.484297:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762935.484306:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762935.484312:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762935.484317:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762935.484324:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762935.484328:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762935.484335:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762935.868252:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762935.868263:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762935.868267:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762935.868276:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762935.868283:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762935.868289:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762935.868294:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762935.868299:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762935.868304:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762936.508274:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762936.508284:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762936.508288:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762936.508296:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762936.508304:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762936.508310:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762936.508314:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762936.508318:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762936.508324:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762936.892249:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762936.892259:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762936.892263:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762936.892283:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762936.892296:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762936.892301:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762936.892308:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762936.892315:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762936.892322:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762937.532236:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762937.532245:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762937.532249:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762937.532257:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762937.532266:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762937.532275:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762937.532282:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762937.532286:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762937.532291:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762937.916265:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762937.916275:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762937.916278:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762937.916284:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762937.916289:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762937.916293:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762937.916297:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762937.916300:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762937.916304:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762938.556269:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762938.556278:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762938.556283:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762938.556295:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762938.556301:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762938.556308:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762938.556317:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762938.556320:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762938.556324:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762938.940280:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762938.940292:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762938.940297:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762938.940309:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762938.940316:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762938.940323:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762938.940332:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762938.940338:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762938.940345:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787762938.941203:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762938.941212:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787762938.942352:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762938.942355:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787762938.942774:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762938.942778:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787762938.943225:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762938.943229:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:0.0:1787762939.580246:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762939.580258:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762939.580262:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762939.580270:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762939.580277:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762939.580281:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762939.580287:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762939.580291:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762939.580296:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762939.964286:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762939.964296:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762939.964299:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762939.964306:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762939.964310:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762939.964315:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762939.964319:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762939.964322:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762939.964325:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762940.604222:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762940.604231:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762940.604234:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762940.604243:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762940.604248:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762940.604253:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762940.604257:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762940.604260:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762940.604264:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762940.988268:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762940.988278:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762940.988282:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762940.988290:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762940.988296:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762940.988302:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762940.988306:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762940.988310:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762940.988314:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762941.628215:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762941.628234:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762941.628237:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762941.628244:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762941.628249:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762941.628254:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762941.628259:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762941.628263:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762941.628269:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762942.012237:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762942.012248:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762942.012252:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762942.012259:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762942.012265:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762942.012269:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762942.012273:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762942.012277:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762942.012280:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762942.652214:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762942.652222:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762942.652226:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762942.652230:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762942.652235:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762942.652239:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762942.652243:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762942.652246:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762942.652253:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762943.036191:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762943.036197:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762943.036199:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762943.036204:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762943.036207:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762943.036211:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762943.036213:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762943.036216:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762943.036218:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762943.676205:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762943.676213:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762943.676215:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762943.676219:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762943.676224:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762943.676228:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762943.676232:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762943.676236:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762943.676239:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762944.060211:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762944.060219:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762944.060221:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762944.060225:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762944.060229:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762944.060232:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762944.060235:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762944.060239:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762944.060249:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787762944.060729:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762944.060734:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762944.061425:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762944.061427:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762944.061642:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762944.061644:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787762944.061836:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762944.061838:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:0.0:1787762944.700291:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762944.700302:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762944.700306:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762944.700312:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762944.700319:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762944.700324:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762944.700329:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762944.700334:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762944.700339:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762945.084236:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762945.084246:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762945.084250:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762945.084257:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762945.084263:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762945.084280:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762945.084285:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762945.084288:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762945.084293:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762945.724239:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762945.724253:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762945.724258:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762945.724267:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762945.724275:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762945.724284:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762945.724295:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762945.724301:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762945.724308:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762946.108286:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762946.108296:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762946.108299:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762946.108307:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762946.108313:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762946.108319:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762946.108323:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762946.108327:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762946.108331:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762946.748278:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762946.748287:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762946.748293:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762946.748302:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762946.748313:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762946.748322:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762946.748327:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762946.748331:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762946.748336:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762947.132235:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762947.132245:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762947.132248:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762947.132255:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762947.132261:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762947.132266:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762947.132270:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762947.132274:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762947.132278:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762947.772294:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762947.772303:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762947.772306:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762947.772312:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762947.772320:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762947.772328:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762947.772334:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762947.772341:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762947.772352:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762948.156233:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762948.156242:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762948.156245:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762948.156251:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762948.156256:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762948.156262:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762948.156266:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762948.156270:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762948.156274:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762948.796289:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762948.796300:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762948.796308:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762948.796316:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762948.796326:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762948.796335:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762948.796347:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762948.796352:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762948.796357:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762949.180287:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762949.180295:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762949.180300:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762949.180308:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762949.180315:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762949.180322:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762949.180328:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762949.180333:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762949.180339:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787762949.181380:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762949.181395:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787762949.182333:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762949.182337:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787762949.182910:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762949.182913:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787762949.183346:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787762949.183349:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000100:00000001:0.0:1787762949.820246:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762949.820257:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762949.820261:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762949.820274:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762949.820283:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762949.820292:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762949.820296:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762949.820302:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762949.820309:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762950.204280:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762950.204292:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762950.204296:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762950.204305:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762950.204311:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762950.204315:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762950.204319:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762950.204323:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762950.204327:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762950.844205:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762950.844215:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762950.844217:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762950.844223:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762950.844228:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762950.844234:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762950.844241:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762950.844245:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762950.844251:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762951.228197:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762951.228208:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762951.228211:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762951.228219:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762951.228227:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762951.228233:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762951.228236:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762951.228239:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762951.228243:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762951.868247:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762951.868258:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762951.868261:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762951.868268:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762951.868273:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762951.868277:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762951.868280:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762951.868283:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762951.868287:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762952.252244:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762952.252254:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762952.252257:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762952.252265:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762952.252270:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762952.252274:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762952.252278:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762952.252281:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762952.252284:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762952.892226:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762952.892234:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762952.892239:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762952.892244:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762952.892251:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762952.892257:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762952.892263:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762952.892268:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762952.892275:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762953.916300:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762953.916311:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762953.916314:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762953.916322:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762953.916329:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762953.916335:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762953.916339:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762953.916342:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762953.916347:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762954.300327:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762954.300337:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762954.300343:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762954.300352:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762954.300360:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762954.300365:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762954.300372:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762954.300378:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762954.300387:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787762954.301267:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762954.301276:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1787762954.302554:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762954.302556:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1787762954.302752:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762954.302754:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1787762954.302980:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762954.302982:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000100:00000001:0.0:1787762954.940193:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762954.940200:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762954.940203:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762954.940210:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762954.940215:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762954.940219:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762954.940222:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762954.940225:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762954.940229:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762955.324293:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762955.324300:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762955.324303:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762955.324309:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762955.324313:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762955.324317:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762955.324320:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762955.324322:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762955.324325:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762955.964240:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762955.964251:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762955.964257:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762955.964265:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762955.964275:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762955.964283:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762955.964288:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762955.964293:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762955.964300:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762956.348226:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762956.348239:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762956.348242:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762956.348252:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787762956.348259:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762956.348264:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762956.348269:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762956.348273:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762956.348278:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762956.988224:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762956.988234:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762956.988238:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762956.988245:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762956.988250:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762956.988255:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762956.988270:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762956.988273:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762956.988276:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762958.012239:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762958.012249:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762958.012251:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762958.012257:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762958.012263:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762958.012267:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762958.012271:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762958.012274:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762958.012278:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762959.036335:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762959.036345:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762959.036350:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762959.036361:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762959.036366:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762959.036371:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762959.036375:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762959.036379:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762959.036383:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787762959.421232:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762959.421856:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762959.422332:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762959.423090:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787762960.060339:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762960.060348:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762960.060353:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762960.060359:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762960.060365:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762960.060370:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762960.060374:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762960.060378:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762960.060383:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762961.084234:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762961.084244:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762961.084247:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762961.084253:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762961.084257:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762961.084263:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762961.084266:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762961.084269:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762961.084272:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762962.108299:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762962.108312:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762962.108317:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762962.108327:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762962.108335:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762962.108342:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762962.108350:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762962.108357:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762962.108364:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762963.132280:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762963.132291:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762963.132295:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762963.132302:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762963.132308:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762963.132314:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762963.132318:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762963.132321:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762963.132326:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762964.156291:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762964.156300:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762964.156306:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762964.156319:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762964.156327:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762964.156338:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762964.156343:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762964.156346:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762964.156350:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787762964.541446:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762964.541460:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787762964.542513:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762964.542517:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787762964.542909:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762964.542913:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787762964.543357:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762964.543388:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:0.0:1787762965.180241:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762965.180254:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762965.180258:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762965.180267:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762965.180279:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762965.180285:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762965.180290:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762965.180294:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762965.180299:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762966.204250:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762966.204259:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762966.204263:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762966.204272:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762966.204282:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762966.204290:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762966.204296:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762966.204300:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762966.204308:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762967.228262:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762967.228272:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762967.228276:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762967.228283:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762967.228290:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762967.228297:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762967.228301:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762967.228304:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762967.228309:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762968.252289:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762968.252301:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762968.252307:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762968.252316:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762968.252326:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762968.252357:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762968.252362:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762968.252369:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762968.252381:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762969.276229:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762969.276236:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762969.276239:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762969.276245:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762969.276249:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762969.276252:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762969.276254:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762969.276256:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762969.276259:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787762969.661481:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762969.661492:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787762969.662396:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762969.662399:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787762969.662774:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762969.662777:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787762969.663112:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762969.663115:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:0.0:1787762970.300262:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762970.300272:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762970.300275:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762970.300282:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762970.300289:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762970.300297:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762970.300305:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762970.300312:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762970.300347:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762971.324245:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762971.324255:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762971.324259:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762971.324266:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762971.324272:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762971.324277:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762971.324281:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762971.324285:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762971.324290:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762972.348337:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762972.348346:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762972.348349:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762972.348355:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762972.348361:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762972.348366:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762972.348369:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762972.348374:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762972.348379:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762973.372273:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762973.372282:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762973.372285:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762973.372292:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762973.372320:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762973.372332:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762973.372339:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762973.372343:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762973.372350:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762974.396281:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762974.396313:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762974.396318:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762974.396325:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762974.396332:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762974.396339:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762974.396348:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762974.396353:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762974.396358:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787762974.780317:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762974.780330:0:259241:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787762974.780335:0:259241:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762974.780342:0:259241:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762974.780354:0:259241:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762974.780365:0:259241:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 20689
00000400:00000200:0.0:1787762974.780385:0:259241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762974.780400:0:259241:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762974.780406:0:259241:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762974.780417:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762974.780428:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762974.780438:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762974.780445:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762974.780449:0:259241:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762974.780452:0:259241:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787762974.780455:0:259241:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762974.780460:0:259241:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787762974.780464:0:259241:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787762974.780468:0:259241:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787762974.780473:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762974.780476:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762974.780479:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762974.780483:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762974.780487:0:259241:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787762974.780491:0:259241:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762974.780496:0:259241:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762974.780500:0:259241:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762974.780504:0:259241:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762974.780509:0:259241:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762974.780511:0:259241:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787762974.780514:0:259241:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762974.780524:0:259241:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
00000020:00000040:0.0:1787762974.780528:0:259241:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787762974.780534:0:259241:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762974.780536:0:259241:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762974.780540:0:259241:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762974.780543:0:259241:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762974.780547:0:259241:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762974.780556:0:259241:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787762974.780565:0:259241:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762974.780577:0:259241:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832578432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762974.780591:0:259241:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762974.780596:0:259241:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762974.780602:0:259241:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832578432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762974.780630:0:259241:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787762974.780634:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762974.780639:0:259241:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787762974.780641:0:259241:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762974.780643:0:259241:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762974.780647:0:259241:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762974.780650:0:259241:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20689
00000400:00000200:0.0:1787762974.780656:0:259241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762974.780661:0:259241:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762974.780665:0:259241:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762974.780670:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762974.780674:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762974.780684:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762974.780693:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762974.780699:0:259241:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762974.780702:0:259241:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787762974.780708:0:259241:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762974.780713:0:259241:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787762974.780723:0:259241:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787762974.780733:0:259241:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787762974.780748:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762974.780756:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762974.780765:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762974.780774:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762974.780787:0:259241:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787762974.780802:0:259241:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762974.780814:0:259241:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762974.780822:0:259241:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762974.780833:0:259241:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762974.780844:0:259241:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762974.780851:0:259241:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787762974.780860:0:259241:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787762974.780871:0:259241:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762974.780881:0:259241:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762974.780888:0:259241:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787762974.780898:0:259241:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787762974.780909:0:259241:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762974.780911:0:259241:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762974.780914:0:259241:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762974.780916:0:259241:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762974.780925:0:259241:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762974.780934:0:259241:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787762974.780945:0:259241:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762974.780959:0:259241:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832578560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762974.780970:0:259241:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762974.780978:0:259241:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762974.780992:0:259241:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832578560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762974.781039:0:259241:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787762974.781049:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762974.781061:0:259241:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787762974.781066:0:259241:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762974.781069:0:259241:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762974.781072:0:259241:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762974.781082:0:259241:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20689
00000400:00000200:0.0:1787762974.781169:0:259241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762974.781178:0:259241:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762974.781184:0:259241:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762974.781199:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762974.781211:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762974.781221:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762974.781228:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762974.781233:0:259241:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762974.781239:0:259241:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787762974.781244:0:259241:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762974.781250:0:259241:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787762974.781258:0:259241:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787762974.781264:0:259241:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787762974.781277:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:1.1:1787762974.781285:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787762974.781287:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762974.781314:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000200:1.1:1787762974.781317:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:0.0:1787762974.781320:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762974.781328:0:259241:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787762974.781340:0:259241:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762974.781349:0:259241:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762974.781355:0:259241:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762974.781365:0:259241:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762974.781374:0:259241:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762974.781379:0:259241:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787762974.781383:0:259241:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787762974.781390:0:259241:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762974.781398:0:259241:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762974.781409:0:259241:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000020:00000040:0.0:1787762974.781417:0:259241:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787762974.781426:0:259241:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762974.781431:0:259241:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762974.781438:0:259241:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762974.781441:0:259241:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762974.781449:0:259241:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762974.781458:0:259241:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787762974.781470:0:259241:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762974.781485:0:259241:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832578688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762974.781507:0:259241:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762974.781512:0:259241:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762974.781524:0:259241:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832578688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762974.781541:0:259241:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762974.781552:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787762974.781559:0:259241:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787762974.781563:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762974.781571:0:259241:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787762974.781574:0:259241:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762974.781577:0:259241:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762974.781589:0:259241:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762974.781594:0:259241:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20689
00000400:00000200:0.0:1787762974.781607:0:259241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762974.781616:0:259241:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762974.781622:0:259241:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762974.781631:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762974.781638:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762974.781644:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762974.781653:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762974.781663:0:259241:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762974.781668:0:259241:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787762974.781673:0:259241:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762974.781680:0:259241:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787762974.781686:0:259241:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787762974.781694:0:259241:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787762974.781706:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762974.781712:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762974.781721:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762974.781727:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762974.781735:0:259241:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787762974.781746:0:259241:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762974.781758:0:259241:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762974.781765:0:259241:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762974.781772:0:259241:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762974.781779:0:259241:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762974.781787:0:259241:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787762974.781793:0:259241:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787762974.781803:0:259241:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762974.781808:0:259241:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762974.781818:0:259241:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787762974.781828:0:259241:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787762974.781835:0:259241:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762974.781842:0:259241:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762974.781849:0:259241:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762974.781855:0:259241:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762974.781861:0:259241:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762974.781872:0:259241:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787762974.781881:0:259241:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762974.781896:0:259241:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832578816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762974.781928:0:259241:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762974.781937:0:259241:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762974.781948:0:259241:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832578816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762974.781978:0:259241:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762974.781985:0:259241:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20699)
00000800:00000200:1.1:1787762974.782315:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762974.782322:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787762974.782766:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762974.782770:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787762974.783129:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762974.783163:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:0.0:1787762975.420338:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762975.420348:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762975.420353:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762975.420360:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762975.420369:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762975.420376:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762975.420387:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762975.420396:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762975.420405:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762976.444309:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762976.444318:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762976.444323:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762976.444333:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762976.444342:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762976.444348:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762976.444352:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762976.444356:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762976.444361:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762977.468198:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762977.468204:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762977.468206:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762977.468210:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762977.468215:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762977.468219:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762977.468222:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762977.468224:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762977.468226:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762978.492216:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762978.492222:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762978.492226:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762978.492230:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762978.492235:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762978.492239:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762978.492242:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762978.492244:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762978.492246:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762979.516201:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762979.516210:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762979.516213:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762979.516220:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762979.516228:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762979.516234:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762979.516238:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762979.516240:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762979.516243:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787762979.900235:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762979.900244:0:259241:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787762979.900247:0:259241:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762979.900252:0:259241:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762979.900267:0:259241:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762979.900276:0:259241:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 20694
00000400:00000200:0.0:1787762979.900285:0:259241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762979.900298:0:259241:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762979.900301:0:259241:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762979.900304:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762979.900307:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762979.900310:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762979.900313:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762979.900317:0:259241:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762979.900319:0:259241:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787762979.900321:0:259241:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762979.900324:0:259241:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787762979.900326:0:259241:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787762979.900328:0:259241:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787762979.900330:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762979.900333:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762979.900335:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762979.900336:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762979.900339:0:259241:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787762979.900342:0:259241:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762979.900343:0:259241:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762979.900345:0:259241:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762979.900348:0:259241:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762979.900350:0:259241:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762979.900352:0:259241:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787762979.900354:0:259241:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762979.900359:0:259241:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787762979.900362:0:259241:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787762979.900365:0:259241:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762979.900366:0:259241:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762979.900368:0:259241:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762979.900370:0:259241:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762979.900373:0:259241:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762979.900379:0:259241:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787762979.900385:0:259241:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762979.900393:0:259241:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832578944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762979.900402:0:259241:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762979.900406:0:259241:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762979.900411:0:259241:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832578944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762979.900427:0:259241:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787762979.900429:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762979.900432:0:259241:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787762979.900436:0:259241:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762979.900438:0:259241:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762979.900442:0:259241:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762979.900446:0:259241:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20694
00000400:00000200:0.0:1787762979.900451:0:259241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762979.900459:0:259241:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762979.900463:0:259241:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762979.900468:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762979.900472:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762979.900476:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762979.900479:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762979.900484:0:259241:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762979.900486:0:259241:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787762979.900490:0:259241:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762979.900494:0:259241:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787762979.900499:0:259241:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787762979.900503:0:259241:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787762979.900509:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762979.900513:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762979.900515:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762979.900518:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762979.900521:0:259241:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787762979.900524:0:259241:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762979.900529:0:259241:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762979.900533:0:259241:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762979.900538:0:259241:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762979.900542:0:259241:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762979.900545:0:259241:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787762979.900549:0:259241:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787762979.900555:0:259241:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762979.900556:0:259241:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762979.900560:0:259241:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787762979.900564:0:259241:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787762979.900570:0:259241:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762979.900572:0:259241:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762979.900575:0:259241:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762979.900578:0:259241:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762979.900581:0:259241:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762979.900586:0:259241:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787762979.900590:0:259241:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762979.900599:0:259241:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832579072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762979.900610:0:259241:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762979.900615:0:259241:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762979.900622:0:259241:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832579072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762979.900632:0:259241:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787762979.900635:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762979.900640:0:259241:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787762979.900643:0:259241:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762979.900645:0:259241:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762979.900649:0:259241:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762979.900655:0:259241:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20694
00000400:00000200:0.0:1787762979.900675:0:259241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762979.900683:0:259241:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762979.900686:0:259241:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762979.900694:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762979.900699:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762979.900704:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762979.900707:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762979.900710:0:259241:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762979.900712:0:259241:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787762979.900713:0:259241:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762979.900716:0:259241:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787762979.900718:0:259241:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787762979.900723:0:259241:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787762979.900727:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762979.900730:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.1:1787762979.900733:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787762979.900734:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762979.900740:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:1.1:1787762979.900741:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000020:00000040:0.0:1787762979.900744:0:259241:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787762979.900750:0:259241:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762979.900756:0:259241:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762979.900761:0:259241:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762979.900764:0:259241:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762979.900769:0:259241:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762979.900773:0:259241:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787762979.900777:0:259241:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787762979.900781:0:259241:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762979.900784:0:259241:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762979.900790:0:259241:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787762979.900793:0:259241:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787762979.900798:0:259241:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762979.900800:0:259241:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762979.900803:0:259241:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762979.900806:0:259241:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762979.900809:0:259241:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762979.900814:0:259241:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787762979.900818:0:259241:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762979.900826:0:259241:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832579200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762979.900844:0:259241:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762979.900848:0:259241:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762979.900852:0:259241:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832579200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762979.900867:0:259241:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762979.900872:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787762979.900877:0:259241:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787762979.900882:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787762979.900889:0:259241:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787762979.900891:0:259241:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787762979.900895:0:259241:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787762979.900902:0:259241:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787762979.900910:0:259241:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20694
00000400:00000200:0.0:1787762979.900919:0:259241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787762979.900928:0:259241:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787762979.900931:0:259241:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787762979.900938:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762979.900940:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762979.900944:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762979.900947:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787762979.900951:0:259241:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787762979.900953:0:259241:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787762979.900955:0:259241:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787762979.900958:0:259241:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787762979.900961:0:259241:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787762979.900966:0:259241:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787762979.900972:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762979.900974:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787762979.900978:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787762979.900983:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787762979.900987:0:259241:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787762979.900992:0:259241:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787762979.900995:0:259241:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787762979.900998:0:259241:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787762979.901002:0:259241:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787762979.901007:0:259241:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762979.901012:0:259241:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787762979.901015:0:259241:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787762979.901019:0:259241:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762979.901022:0:259241:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787762979.901026:0:259241:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000020:00000040:0.0:1787762979.901031:0:259241:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787762979.901035:0:259241:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787762979.901038:0:259241:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762979.901042:0:259241:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787762979.901045:0:259241:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787762979.901047:0:259241:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787762979.901052:0:259241:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787762979.901056:0:259241:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762979.901064:0:259241:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832579328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787762979.901079:0:259241:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787762979.901083:0:259241:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762979.901090:0:259241:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832579328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787762979.901106:0:259241:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787762979.901113:0:259241:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20704)
00000800:00000200:1.1:1787762979.901286:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762979.901288:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787762979.901482:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762979.901484:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787762979.901660:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762979.901661:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:0.0:1787762980.540320:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762980.540329:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762980.540334:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762980.540343:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762980.540349:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762980.540354:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762980.540359:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762980.540366:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762980.540371:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762981.564205:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762981.564212:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762981.564216:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762981.564220:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762981.564225:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762981.564229:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762981.564232:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762981.564234:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762981.564236:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762982.588220:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762982.588227:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762982.588230:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762982.588234:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762982.588253:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762982.588256:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762982.588259:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762982.588262:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762982.588265:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762983.612211:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762983.612218:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762983.612221:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762983.612228:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762983.612244:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762983.612249:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762983.612254:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762983.612259:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762983.612262:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762984.636197:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762984.636205:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762984.636209:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762984.636213:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762984.636219:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762984.636237:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762984.636241:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762984.636243:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762984.636245:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787762985.020750:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762985.020756:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787762985.021297:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762985.021299:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787762985.021504:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762985.021506:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787762985.021683:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762985.021685:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:0.0:1787762985.660300:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762985.660324:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762985.660327:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762985.660333:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762985.660339:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762985.660347:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762985.660356:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762985.660362:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762985.660368:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762986.684204:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762986.684224:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762986.684228:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762986.684234:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762986.684239:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762986.684245:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762986.684249:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762986.684252:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762986.684256:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762987.708229:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762987.708239:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762987.708242:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762987.708260:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762987.708266:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762987.708270:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762987.708274:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762987.708278:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762987.708281:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762988.732216:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762988.732223:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762988.732225:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762988.732230:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762988.732236:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762988.732240:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762988.732243:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762988.732245:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762988.732251:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762989.756220:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762989.756229:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762989.756233:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762989.756241:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762989.756248:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762989.756253:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762989.756256:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762989.756259:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762989.756264:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787762990.140764:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762990.140771:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787762990.141271:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762990.141272:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787762990.141497:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762990.141498:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787762990.141679:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762990.141681:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000100:00000001:0.0:1787762990.780211:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762990.780219:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762990.780221:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762990.780226:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762990.780229:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762990.780233:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762990.780237:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762990.780241:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762990.780247:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762991.804220:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762991.804227:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762991.804231:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762991.804235:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762991.804239:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762991.804243:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762991.804246:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762991.804250:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762991.804253:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762992.828242:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762992.828250:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762992.828252:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762992.828257:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762992.828263:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762992.828267:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762992.828270:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762992.828273:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762992.828276:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762993.852281:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762993.852292:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762993.852298:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762993.852306:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762993.852315:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762993.852320:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762993.852324:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762993.852327:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762993.852330:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762994.876251:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762994.876261:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762994.876265:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762994.876271:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762994.876278:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762994.876284:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762994.876289:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762994.876293:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762994.876297:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787762995.261417:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762995.261436:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1787762995.262321:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762995.262325:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1787762995.262752:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762995.262756:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:1.1:1787762995.263127:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787762995.263132:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000100:00000001:0.0:1787762995.900278:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762995.900289:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762995.900293:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762995.900311:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762995.900319:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762995.900324:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762995.900328:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762995.900332:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762995.900338:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762996.924213:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762996.924223:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762996.924225:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762996.924231:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762996.924239:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762996.924243:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762996.924248:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762996.924254:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762996.924258:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762997.948218:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762997.948230:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762997.948235:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762997.948243:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762997.948250:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762997.948257:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762997.948265:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762997.948270:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762997.948286:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762998.972248:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762998.972260:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762998.972266:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762998.972273:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762998.972285:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762998.972293:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762998.972298:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762998.972303:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762998.972307:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787762999.996264:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787762999.996277:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787762999.996280:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787762999.996288:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787762999.996295:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787762999.996304:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762999.996323:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787762999.996330:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787762999.996354:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787763000.381279:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763000.381291:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787763000.382985:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763000.382999:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787763000.383520:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763000.383522:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787763000.383755:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763000.383757:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:0.0:1787763001.020288:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763001.020299:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763001.020303:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763001.020314:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763001.020323:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763001.020330:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763001.020336:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763001.020342:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763001.020354:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763002.044206:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763002.044216:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763002.044220:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763002.044225:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763002.044233:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763002.044240:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763002.044243:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763002.044246:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763002.044249:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763003.068283:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763003.068297:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763003.068302:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763003.068316:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763003.068332:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763003.068345:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763003.068370:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763003.068388:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763003.068412:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787763005.501361:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763005.501371:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787763005.502365:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763005.502369:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787763005.502795:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763005.502799:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787763005.503314:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763005.503319:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787763010.620847:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763010.620854:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787763010.621341:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763010.621343:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787763010.621551:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763010.621552:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787763010.621737:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763010.621739:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787763015.740967:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763015.740970:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787763015.741301:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763015.741303:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787763015.741507:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763015.741509:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787763015.741690:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763015.741691:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:0.0:1787763017.404379:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763017.404383:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763017.404386:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763020.476280:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763020.476285:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763020.476287:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763020.860230:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763020.860237:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763020.860239:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763020.860244:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763020.860249:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763020.860252:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763020.860254:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763020.860256:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763020.860258:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787763020.860858:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763020.860865:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787763020.861248:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763020.861250:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787763020.861442:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763020.861444:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787763020.861617:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763020.861618:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:0.0:1787763021.884265:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763021.884274:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763021.884278:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763021.884286:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763021.884293:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763021.884299:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763021.884303:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763021.884307:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763021.884312:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763025.980415:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763025.980435:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763025.980446:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763025.980459:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763025.980474:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763025.980488:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 20740
00000400:00000200:0.0:1787763025.980502:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763025.980515:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763025.980521:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763025.980526:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763025.980531:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763025.980534:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763025.980538:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763025.980543:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763025.980545:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763025.980549:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763025.980553:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763025.980557:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763025.980560:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763025.980564:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763025.980567:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763025.980569:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763025.980572:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763025.980575:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763025.980580:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763025.980583:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763025.980586:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763025.980593:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763025.980596:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763025.980599:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763025.980603:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763025.980613:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763025.980618:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763025.980623:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763025.980625:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763025.980629:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763025.980632:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763025.980637:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763025.980647:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763025.980654:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763025.980666:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832583552/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763025.980687:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763025.980694:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763025.980705:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832583552/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763025.980747:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763025.980755:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763025.980762:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763025.980767:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763025.980771:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763025.980777:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763025.980783:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20740
00000400:00000200:0.0:1787763025.980790:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763025.980795:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763025.980798:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763025.980802:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763025.980805:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763025.980809:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763025.980811:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763025.980814:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763025.980816:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763025.980819:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763025.980823:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763025.980826:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763025.980829:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763025.980834:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763025.980837:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763025.980840:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763025.980843:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763025.980846:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763025.980850:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763025.980854:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763025.980861:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763025.980869:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763025.980878:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763025.980882:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763025.980891:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763025.980901:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763025.980908:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763025.980918:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000020:00000040:0.0:1787763025.980925:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763025.980932:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763025.980958:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763025.980965:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763025.980975:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763025.980983:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763025.980993:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763025.981006:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763025.981023:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832583680/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763025.981046:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763025.981053:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763025.981063:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832583680/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763025.981090:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763025.981098:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763025.981109:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763025.981111:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763025.981114:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763025.981188:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763025.981199:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20740
00000400:00000200:0.0:1787763025.981213:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763025.981223:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763025.981234:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763025.981248:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763025.981252:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763025.981261:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763025.981267:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763025.981273:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763025.981277:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763025.981285:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763025.981292:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763025.981300:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763025.981307:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763025.981319:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763025.981327:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763025.981338:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763025.981347:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763025.981356:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763025.981366:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763025.981374:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763025.981377:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763025.981384:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763025.981392:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763025.981402:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763025.981411:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763025.981422:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763025.981432:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763025.981445:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
00000800:00000200:2.1:1787763025.981456:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:0.0:1787763025.981458:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763025.981468:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763025.981471:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763025.981475:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763025.981479:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763025.981484:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763025.981500:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787763025.981512:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763025.981526:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832583808/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763025.981544:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763025.981551:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763025.981561:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832583808/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763025.981588:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763025.981596:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763025.981604:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763025.981611:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763025.981617:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763025.981625:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763025.981630:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763025.981637:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763025.981647:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20740
00000400:00000200:0.0:1787763025.981657:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763025.981672:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763025.981684:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763025.981701:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763025.981714:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763025.981730:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763025.981740:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763025.981753:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763025.981760:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763025.981769:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763025.981778:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763025.981786:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763025.981790:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763025.981801:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763025.981810:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763025.981815:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763025.981826:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763025.981834:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763025.981846:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763025.981855:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763025.981864:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763025.981872:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763025.981885:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763025.981891:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763025.981899:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763025.981905:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763025.981914:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763025.981928:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000020:00000040:0.0:1787763025.981965:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763025.981980:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763025.981987:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763025.981998:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763025.982007:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763025.982017:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763025.982035:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763025.982049:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763025.982068:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832583936/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763025.982119:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:2.1:1787763025.982130:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763025.982131:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763025.982194:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832583936/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763025.982236:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763025.982261:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20750)
00000800:00000200:2.1:1787763025.982566:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763025.982906:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787763026.620406:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763026.620413:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763026.620419:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000008:00000020:0.0:1787763027.516271:0:304413:0:(osc_request.c:747:osc_update_next_shrink()) next time 21947 to shrink grant
00000008:00000020:0.0:1787763027.516282:0:304413:0:(osc_request.c:747:osc_update_next_shrink()) next time 21947 to shrink grant
00000008:00000020:0.0:1787763027.516285:0:304413:0:(osc_request.c:747:osc_update_next_shrink()) next time 21947 to shrink grant
00000020:00000001:0.0:1787763029.692317:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763029.692322:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763029.692326:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1787763031.100970:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763031.100978:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787763031.101655:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763031.101658:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787763031.102051:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763031.102055:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787763031.102417:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763031.102420:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:0.0:1787763032.764370:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763032.764376:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763032.764380:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763035.836334:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763035.836343:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763035.836349:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763036.220346:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763036.220364:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763036.220371:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763036.220376:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763036.220383:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763036.220396:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 20750
00000400:00000200:0.0:1787763036.220407:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763036.220416:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763036.220421:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763036.220426:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763036.220431:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763036.220435:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763036.220439:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763036.220444:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763036.220446:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763036.220450:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763036.220454:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763036.220457:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763036.220461:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763036.220464:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763036.220467:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763036.220470:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763036.220472:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763036.220476:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763036.220480:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763036.220483:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763036.220487:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763036.220491:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763036.220495:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763036.220498:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763036.220501:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763036.220512:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763036.220517:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763036.220522:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763036.220524:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763036.220528:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763036.220532:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763036.220537:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763036.220546:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787763036.220554:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763036.220565:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832584576/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763036.220579:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763036.220584:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763036.220590:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832584576/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763036.220619:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763036.220624:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763036.220629:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763036.220631:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763036.220633:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763036.220636:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763036.220640:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20750
00000400:00000200:0.0:1787763036.220645:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763036.220649:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763036.220652:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763036.220657:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763036.220661:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763036.220663:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763036.220668:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763036.220672:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763036.220677:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763036.220680:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763036.220687:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763036.220694:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763036.220697:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763036.220705:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763036.220713:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763036.220719:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763036.220726:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763036.220734:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763036.220744:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763036.220750:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763036.220755:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763036.220765:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763036.220773:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763036.220779:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763036.220787:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763036.220796:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763036.220803:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763036.220811:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763036.220820:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763036.220828:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763036.220833:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763036.220835:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763036.220839:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763036.220845:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763036.220855:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787763036.220892:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763036.220908:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832584704/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763036.220927:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763036.220934:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763036.220944:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832584704/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763036.221026:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763036.221034:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763036.221042:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763036.221051:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763036.221059:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763036.221066:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763036.221076:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20750
00000400:00000200:0.0:1787763036.221090:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763036.221103:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763036.221107:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763036.221122:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763036.221132:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763036.221185:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763036.221195:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763036.221207:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763036.221215:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763036.221225:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763036.221232:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763036.221239:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763036.221247:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763036.221260:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763036.221272:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763036.221286:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763036.221297:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763036.221308:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763036.221327:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763036.221341:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787763036.221347:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:0.0:1787763036.221352:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:2.1:1787763036.221355:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
02000000:00000001:0.0:1787763036.221360:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763036.221372:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763036.221382:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763036.221388:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763036.221396:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763036.221407:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763036.221417:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763036.221424:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763036.221432:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763036.221437:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763036.221441:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763036.221448:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763036.221455:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763036.221469:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787763036.221481:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763036.221496:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832584832/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763036.221554:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763036.221560:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763036.221573:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832584832/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763036.221597:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763036.221605:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763036.221615:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763036.221626:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763036.221636:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763036.221642:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763036.221647:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763036.221653:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763036.221661:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20750
00000400:00000200:0.0:1787763036.221671:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763036.221682:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763036.221687:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763036.221700:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763036.221708:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763036.221715:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763036.221720:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763036.221729:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763036.221734:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763036.221739:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763036.221746:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763036.221749:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763036.221756:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763036.221770:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763036.221779:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763036.221788:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763036.221794:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763036.221803:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763036.221810:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763036.221819:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763036.221827:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763036.221835:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763036.221844:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763036.221850:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763036.221857:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763036.221888:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763036.221897:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763036.221906:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787763036.221916:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763036.221926:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763036.221931:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763036.221936:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763036.221943:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763036.221950:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763036.221960:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763036.221969:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763036.221988:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832584960/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763036.222007:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763036.222014:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763036.222029:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832584960/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763036.222068:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763036.222077:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20760)
00000800:00000200:2.1:1787763036.222389:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763036.222393:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787763036.222818:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763036.222822:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787763036.223242:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763036.223245:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:0.0:1787763038.908280:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763038.908283:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763038.908286:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.1:1787763041.341287:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763041.341295:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787763041.342210:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763041.342213:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787763041.342583:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763041.342587:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787763041.342951:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763041.342955:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000020:00000001:0.0:1787763041.980404:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763041.980410:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763041.980415:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763045.052317:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763045.052322:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763045.052326:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763046.076298:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763046.076304:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763046.076309:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.1:1787763046.460874:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763046.460883:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787763046.461325:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763046.461327:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787763046.461558:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763046.461560:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787763046.461751:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763046.461755:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000020:00000001:0.0:1787763048.124346:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763048.124350:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763048.124353:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763050.172379:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763050.172388:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763050.172395:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787763051.581203:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763051.581206:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787763051.581549:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763051.581551:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787763051.581751:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763051.581753:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787763051.581980:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763051.581982:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000020:00000001:0.0:1787763053.244274:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763053.244280:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763053.244288:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763056.316332:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763056.316336:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763056.316339:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787763056.700962:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763056.700970:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787763056.701532:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763056.701534:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787763056.701817:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763056.701820:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787763056.702104:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763056.702107:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:0.0:1787763059.388276:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763059.388280:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763059.388284:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787763061.821276:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763061.821285:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787763061.822435:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763061.822437:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787763061.822638:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763061.822640:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787763061.822896:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763061.822898:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:0.0:1787763062.460264:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763062.460267:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763062.460270:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763065.532262:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763065.532266:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763065.532269:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787763066.941995:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763066.941999:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787763066.942374:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763066.942376:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787763066.942633:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763066.942635:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787763066.942880:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763066.942882:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:0.0:1787763068.604340:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763068.604344:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763068.604347:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763070.652331:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763070.652336:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763070.652340:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763071.676322:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763071.676327:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763071.676330:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787763072.060747:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763072.060752:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787763072.061217:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763072.061219:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787763072.061410:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763072.061412:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787763072.061712:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763072.061715:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:0.0:1787763073.724315:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763073.724319:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763073.724321:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763074.748285:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763074.748289:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763074.748291:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763076.796292:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763076.796296:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763076.796298:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787763077.180739:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763077.180745:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787763077.181213:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763077.181215:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787763077.181406:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763077.181408:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787763077.181581:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763077.181583:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:0.0:1787763077.820299:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763077.820302:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763077.820305:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763078.204317:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763078.204323:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763078.204324:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763078.204328:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763078.204331:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763078.204334:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763078.204337:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763078.204339:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763078.204341:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763079.228225:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763079.228233:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763079.228235:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763079.228240:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763079.228243:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763079.228246:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763079.228248:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763079.228251:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763079.228253:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763079.868308:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763079.868312:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763079.868316:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763080.252239:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763080.252246:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763080.252248:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763080.252252:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763080.252256:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763080.252259:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763080.252261:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763080.252263:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763080.252266:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763080.892306:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763080.892310:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763080.892313:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763081.276207:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763081.276213:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763081.276215:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763081.276219:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763081.276223:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763081.276226:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763081.276227:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763081.276230:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763081.276232:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763082.300212:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763082.300221:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763082.300224:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763082.300229:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763082.300233:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763082.300237:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763082.300241:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763082.300245:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763082.300249:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763082.300751:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763082.300754:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787763082.301236:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763082.301238:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787763082.301461:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763082.301463:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787763082.301668:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763082.301670:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000020:00000001:0.0:1787763082.940300:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763082.940303:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763082.940307:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763083.964305:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763083.964308:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763083.964311:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763086.012326:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763086.012330:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763086.012333:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763087.036307:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763087.036311:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763087.036313:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763087.420245:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763087.420254:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763087.420258:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763087.420264:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763087.420271:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763087.420276:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.420282:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763087.420288:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.420295:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763087.420533:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763087.420537:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763087.420538:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763087.420545:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bed00 x1874599832589696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.420552:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763087.420555:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763087.420559:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763087.420564:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763087.420567:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763087.420572:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832589696:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763087.420577:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763087.420582:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763087.420584:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763087.420586:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763087.420591:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9014800.
02000000:00000001:0.0:1787763087.420593:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763087.420599:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763087.420602:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763087.420609:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763087.420616:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832589696, portal 25
00000100:00000001:0.0:1787763087.420618:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763087.420621:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049857280 : -115096659694336 : ffff9751f87bed00)
00000100:00000040:0.0:1787763087.420628:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bed00 x1874599832589696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.420637:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763087.420640:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763087.420644:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763087.420648:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832589696, offset 0
00000400:00000200:0.0:1787763087.420656:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763087.420664:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763087.420670:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763087.420675:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763087.420679:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763087.420683:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763087.420685:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763087.420688:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57842:57842:256:4294967295] 192.168.203.110@tcp LPNI seq info [57842:57842:8:4294967295]
00000400:00000200:0.0:1787763087.420691:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763087.420700:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763087.420706:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763087.420710:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763087.420716:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763087.420720:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787763087.420730:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763087.420736:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1787763087.420742:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763087.420745:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763087.420749:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763087.420753:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763087.420756:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763087.420758:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763087.420762:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763087.420766:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bed00 x1874599832589696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.420775:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763087.420779:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bed00 x1874599832589696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763112 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.420791:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763087.420795:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763087.420811:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763087.420819:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763087.420824:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315475699
00000100:00000010:0.0:1787763087.420831:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c120.
00000100:00000001:0.0:1787763087.420840:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763087.420848:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315475699
00000100:00000010:0.0:1787763087.420855:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c780.
00000100:00000001:0.0:1787763087.420862:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787763087.420867:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763087.420871:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763087.420873:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763087.420880:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763087/real 1787763087]  req@ffff9751f87bed00 x1874599832589696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763087.420906:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763087.420909:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763087.420913:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763087.420919:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763087.420923:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763087.420925:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763087.420930:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bed00 x1874599832589696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763087.420943:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bed00 x1874599832589696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.420963:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763087.420967:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.420971:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763087.420974:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763087.420980:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bed00 x1874599832589696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.420997:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763087.421006:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bed00 x1874599832589696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.421019:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763087.421023:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.421026:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763087.421029:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.421034:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763087.421037:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763087.421040:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763087.421045:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763087.421049:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763087.421059:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bed00 x1874599832589696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763087.421074:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832589696:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763087.421080:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.421084:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763087.421089:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bed00 x1874599832589696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.421100:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763087.421104:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763087.421109:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9014800.
02000000:00000001:0.0:1787763087.421113:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763087.421117:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763087.421120:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787763087.421123:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763087.421128:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612c00.
02000000:00000001:0.0:1787763087.421167:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763087.421170:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763087.421174:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bed00.
00000100:00000001:0.0:1787763087.421181:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763087.421185:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.421190:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.421200:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763087.421204:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763087.421211:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763087.421215:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763087.421219:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763087.421229:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bfc00 x1874599832589824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.421244:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763087.421247:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763087.421254:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763087.421259:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763087.421265:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763087.421274:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832589824:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763087.421283:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763087.421288:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763087.421291:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763087.421296:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763087.421301:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9014800.
02000000:00000001:0.0:1787763087.421307:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763087.421314:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763087.421318:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763087.421323:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763087.421328:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832589824, portal 10
00000100:00000001:0.0:1787763087.421333:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763087.421335:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049861120 : -115096659690496 : ffff9751f87bfc00)
00000100:00000040:0.0:1787763087.421345:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bfc00 x1874599832589824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.421359:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763087.421365:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763087.421368:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763087.421372:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832589824, offset 0
00000400:00000200:0.0:1787763087.421379:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763087.421386:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763087.421393:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763087.421398:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763087.421402:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763087.421406:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763087.421413:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763087.421419:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57843:57843:256:4294967295] 192.168.203.110@tcp LPNI seq info [57843:57843:8:4294967295]
00000400:00000200:0.0:1787763087.421426:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763087.421438:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763087.421447:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763087.421451:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763087.421459:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763087.421463:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787763087.421471:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763087.421475:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1787763087.421483:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763087.421488:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763087.421494:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763087.421501:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763087.421507:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763087.421509:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763087.421530:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763087.421537:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bfc00 x1874599832589824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.421552:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763087.421557:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bfc00 x1874599832589824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763112 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.421568:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763087.421571:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763087.421576:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763087.421580:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:0.0:1787763087.421586:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763087.421590:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315474464
00000100:00000001:0.0:1787763087.421594:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315475699
00000100:00000001:0.0:1787763087.421597:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315474464
00000100:00000001:0.0:1787763087.421601:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315475699
00000100:00000001:0.0:1787763087.421606:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763087.421610:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763087.421615:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763087.421621:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763087/real 1787763087]  req@ffff9751f87bfc00 x1874599832589824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763087.421638:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763087.421642:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763087.421646:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763087.421654:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763087.421659:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763087.421664:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763087.421671:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bfc00 x1874599832589824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763087.421684:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bfc00 x1874599832589824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.421694:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763087.421696:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.421697:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763087.421699:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763087.421702:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bfc00 x1874599832589824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.421706:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763087.421710:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bfc00 x1874599832589824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.421714:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763087.421716:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.421717:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763087.421718:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.421720:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763087.421727:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763087.421729:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763087.421732:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763087.421733:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763087.421736:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bfc00 x1874599832589824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763087.421741:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832589824:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763087.421744:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.421746:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763087.421747:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bfc00 x1874599832589824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.421752:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763087.421753:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763087.421754:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9014800.
02000000:00000001:0.0:1787763087.421756:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763087.421757:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763087.421758:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763087.421759:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763087.421760:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612800.
02000000:00000001:0.0:1787763087.421762:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763087.421763:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763087.421764:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bfc00.
00000100:00000001:0.0:1787763087.421765:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763087.421766:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.421767:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.421770:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763087.421771:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763087.421773:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763087.421774:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763087.421775:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763087.421777:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc3c0 x1874599832589952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.421782:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763087.421783:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763087.421784:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763087.421786:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763087.421787:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763087.421789:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832589952:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763087.421791:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763087.421792:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763087.421794:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763087.421795:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763087.421797:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9014800.
02000000:00000001:0.0:1787763087.421798:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763087.421801:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763087.421802:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763087.421804:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763087.421807:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832589952, portal 10
00000100:00000001:0.0:1787763087.421808:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763087.421809:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049846720 : -115096659704896 : ffff9751f87bc3c0)
00000100:00000040:0.0:1787763087.421813:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc3c0 x1874599832589952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.421818:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763087.421819:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763087.421821:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763087.421822:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832589952, offset 0
00000400:00000200:0.0:1787763087.421825:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763087.421828:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763087.421830:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763087.421832:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763087.421834:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763087.421835:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763087.421837:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763087.421839:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57844:57844:256:4294967295] 192.168.203.110@tcp LPNI seq info [57844:57844:8:4294967295]
00000400:00000200:0.0:1787763087.421842:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763087.421846:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763087.421849:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763087.421851:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763087.421854:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763087.421856:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787763087.421859:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763087.421861:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1787763087.421863:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763087.421865:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763087.421867:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763087.421869:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763087.421871:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763087.421872:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763087.421874:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763087.421876:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc3c0 x1874599832589952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.421881:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763087.421883:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc3c0 x1874599832589952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763112 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.421887:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763087.421888:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763087.421890:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763087.421892:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:0.0:1787763087.421894:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763087.421895:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315474452
00000100:00000001:0.0:1787763087.421897:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315475699
00000100:00000001:0.0:1787763087.421898:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315474452
00000100:00000001:0.0:1787763087.421900:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315475699
00000100:00000001:0.0:1787763087.421902:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763087.421903:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763087.421904:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763087.421906:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763087/real 1787763087]  req@ffff9751f87bc3c0 x1874599832589952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763087.421911:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763087.421912:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763087.421913:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763087.421915:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763087.421916:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763087.421918:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763087.421920:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc3c0 x1874599832589952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763087.421925:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc3c0 x1874599832589952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.421929:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763087.421930:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.421931:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763087.421932:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763087.421934:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc3c0 x1874599832589952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.421938:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763087.421940:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc3c0 x1874599832589952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.421944:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763087.421945:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.421946:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763087.421947:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.421949:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763087.421950:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763087.421953:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763087.421956:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763087.421958:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763087.421962:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc3c0 x1874599832589952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763087.421970:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832589952:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763087.421975:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763087.421979:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc780 x1874599832590080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.421986:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763087.421988:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763087.421990:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763087.421992:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763087.421995:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763087.421998:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832590080:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763087.422001:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763087.422003:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763087.422005:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763087.422006:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763087.422009:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016000.
02000000:00000001:0.0:1787763087.422010:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763087.422013:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763087.422014:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763087.422017:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763087.422019:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832590080, portal 4
00000100:00000001:0.0:1787763087.422020:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763087.422021:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049847680 : -115096659703936 : ffff9751f87bc780)
00000100:00000040:0.0:1787763087.422026:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc780 x1874599832590080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.422030:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763087.422032:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763087.422033:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763087.422035:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832590080, offset 0
00000400:00000200:0.0:1787763087.422037:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763087.422040:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763087.422042:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763087.422044:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763087.422046:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763087.422048:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763087.422049:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763087.422052:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57845:57845:256:4294967295] 192.168.203.110@tcp LPNI seq info [57845:57845:8:4294967295]
00000400:00000200:0.0:1787763087.422054:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763087.422059:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763087.422062:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763087.422064:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763087.422067:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763087.422069:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:0.1:1787763087.422072:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763087.422073:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:0.0:1787763087.422076:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763087.422077:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763087.422079:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763087.422081:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763087.422083:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763087.422085:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763087.422086:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763087.422088:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc780 x1874599832590080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763112 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.422094:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763087.422096:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc780 x1874599832590080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763112 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.422100:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763087.422101:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763087.422103:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763087.422114:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:0.0:1787763087.422117:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763087.422118:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315474469
00000100:00000001:0.0:1787763087.422120:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315475699
00000100:00000001:0.0:1787763087.422121:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315474469
00000100:00000001:0.0:1787763087.422123:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315475699
00000100:00000001:0.0:1787763087.422124:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763087.422126:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763087.422127:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763087.422130:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763087/real 1787763087]  req@ffff9751f87bc780 x1874599832590080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763087.422154:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763087.422155:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763087.422157:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763087.422159:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763087.422160:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763087.422161:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763087.422165:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc780 x1874599832590080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763087.422172:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc780 x1874599832590080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.422178:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763087.422180:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.422183:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763087.422184:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763087.422188:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc780 x1874599832590080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.422196:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763087.422199:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc780 x1874599832590080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.422207:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763087.422210:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.422211:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763087.422213:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.422215:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763087.422217:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763087.422220:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763087.422224:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763087.422227:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763087.422231:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc780 x1874599832590080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763087.422236:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832590080:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763087.422239:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.422240:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763087.422242:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc3c0 x1874599832589952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.422247:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763087.422249:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763087.422251:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9014800.
02000000:00000001:0.0:1787763087.422254:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763087.422255:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763087.422257:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763087.422259:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763087.422260:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612400.
02000000:00000001:0.0:1787763087.422263:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763087.422264:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763087.422266:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc3c0.
00000100:00000001:0.0:1787763087.422268:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763087.422269:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.422271:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763087.422275:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc780 x1874599832590080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763112 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763087.422282:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763087.422284:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763087.422285:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016000.
02000000:00000001:0.0:1787763087.422289:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763087.422290:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763087.422291:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787763087.422293:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763087.422295:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613000.
02000000:00000001:0.0:1787763087.422297:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763087.422298:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763087.422299:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc780.
00000100:00000001:0.0:1787763087.422300:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763087.422301:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.422302:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763087.422305:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763087.422306:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763087.422307:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763087.422308:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763088.444243:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763088.444253:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763088.444257:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763088.444265:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763088.444272:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763088.444277:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763088.444281:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763088.444285:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763088.444289:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763089.084412:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763089.084419:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763089.084427:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763089.468251:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763089.468263:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763089.468265:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763089.468273:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763089.468278:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763089.468282:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763089.468285:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763089.468289:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763089.468292:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763090.108352:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763090.108358:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763090.108363:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763090.492221:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763090.492230:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763090.492232:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763090.492239:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763090.492243:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763090.492247:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763090.492250:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763090.492253:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763090.492255:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763091.516280:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763091.516289:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763091.516291:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763091.516297:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763091.516300:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763091.516304:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763091.516307:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763091.516310:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763091.516312:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763092.540237:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763092.540248:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763092.540254:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763092.540262:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763092.540270:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763092.540283:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 20807
00000400:00000200:0.0:1787763092.540295:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763092.540305:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763092.540309:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763092.540313:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763092.540317:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763092.540321:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763092.540324:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763092.540329:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763092.540332:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763092.540336:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763092.540341:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763092.540344:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763092.540347:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763092.540352:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763092.540355:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763092.540357:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763092.540360:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763092.540364:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763092.540369:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763092.540372:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763092.540376:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763092.540381:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763092.540384:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763092.540387:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763092.540391:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763092.540400:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000020:00000040:0.0:1787763092.540406:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763092.540411:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763092.540413:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763092.540417:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763092.540421:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763092.540427:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763092.540438:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763092.540447:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763092.540459:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832590208/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763092.540473:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763092.540493:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763092.540499:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832590208/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763092.540524:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763092.540528:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763092.540533:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763092.540536:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763092.540539:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763092.540543:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763092.540547:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20807
00000400:00000200:0.0:1787763092.540554:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763092.540561:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763092.540565:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763092.540572:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763092.540578:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763092.540583:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763092.540588:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763092.540595:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763092.540598:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763092.540602:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763092.540607:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763092.540612:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763092.540619:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763092.540626:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763092.540631:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763092.540636:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763092.540640:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763092.540644:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763092.540649:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763092.540654:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763092.540658:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763092.540666:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763092.540670:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763092.540673:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763092.540681:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763092.540686:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763092.540691:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763092.540697:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763092.540704:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763092.540709:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763092.540712:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763092.540718:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763092.540722:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763092.540727:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763092.540737:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787763092.540744:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763092.540756:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832590336/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763092.540772:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763092.540777:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763092.540787:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832590336/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763092.540808:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763092.540814:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763092.540822:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763092.540824:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763092.540828:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763092.540834:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763092.540841:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20807
00000400:00000200:0.0:1787763092.540852:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763092.540860:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763092.540866:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000200:3.1:1787763092.540869:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763092.540874:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763092.540878:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763092.540881:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763092.540883:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763092.540887:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763092.540891:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763092.540895:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763092.540899:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763092.540905:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763092.540911:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763092.540917:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763092.540922:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763092.540925:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763092.540930:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763092.540937:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763092.540946:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763092.540952:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763092.540957:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763092.540962:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763092.540968:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763092.540975:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763092.540980:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763092.540986:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763092.540992:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763092.541001:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
00000020:00000040:0.0:1787763092.541007:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763092.541027:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763092.541031:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763092.541037:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763092.541041:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763092.541046:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763092.541053:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763092.541060:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763092.541072:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832590464/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763092.541103:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763092.541108:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763092.541121:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832590464/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763092.541175:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763092.541181:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763092.541190:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763092.541195:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763092.541203:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763092.541207:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763092.541211:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763092.541216:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763092.541223:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20807
00000400:00000200:0.0:1787763092.541230:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:3.1:1787763092.541235:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000400:00000200:0.0:1787763092.541238:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763092.541245:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763092.541253:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763092.541259:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763092.541265:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763092.541281:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763092.541288:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763092.541292:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763092.541296:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763092.541301:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763092.541306:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763092.541312:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763092.541319:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763092.541325:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763092.541333:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763092.541349:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763092.541355:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763092.541362:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763092.541369:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763092.541374:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763092.541381:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763092.541386:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763092.541390:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763092.541395:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763092.541401:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763092.541405:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763092.541412:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787763092.541419:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763092.541424:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763092.541428:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763092.541434:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763092.541438:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763092.541441:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763092.541451:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763092.541458:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763092.541474:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832590592/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763092.541510:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763092.541519:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763092.541530:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832590592/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763092.541552:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763092.541577:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20817)
00000800:00000200:3.1:1787763092.541614:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763092.541849:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787763093.180341:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763093.180347:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763093.180352:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763094.588247:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763094.588260:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763094.588265:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763094.588271:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763094.588276:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763094.588284:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763094.588287:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763094.588290:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763094.588293:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763095.612272:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763095.612282:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763095.612285:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763095.612293:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763095.612299:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763095.612306:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763095.612310:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763095.612314:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763095.612318:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763096.252509:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763096.252516:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763096.252521:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763096.636230:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763096.636240:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763096.636243:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763096.636249:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763096.636255:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763096.636259:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763096.636262:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763096.636266:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763096.636268:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763097.660258:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763097.660273:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763097.660277:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763097.660282:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763097.660289:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763097.660299:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 20812
00000400:00000200:0.0:1787763097.660313:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763097.660321:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763097.660325:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763097.660329:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763097.660332:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763097.660335:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763097.660338:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763097.660342:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763097.660344:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763097.660347:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763097.660352:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763097.660354:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763097.660357:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763097.660360:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763097.660362:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763097.660363:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763097.660365:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763097.660368:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763097.660371:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763097.660373:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763097.660376:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763097.660380:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763097.660383:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763097.660386:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763097.660389:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763097.660397:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763097.660401:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763097.660405:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763097.660407:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763097.660410:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763097.660414:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763097.660420:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763097.660429:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763097.660439:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763097.660468:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832590720/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763097.660491:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763097.660504:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763097.660529:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832590720/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763097.660574:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763097.660582:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763097.660587:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763097.660590:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763097.660593:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763097.660598:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763097.660603:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20812
00000400:00000200:0.0:1787763097.660609:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763097.660612:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763097.660615:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763097.660618:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763097.660621:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763097.660623:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763097.660626:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763097.660629:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763097.660631:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763097.660633:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763097.660635:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763097.660637:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763097.660641:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763097.660645:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763097.660647:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763097.660650:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763097.660653:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763097.660656:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763097.660659:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763097.660662:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763097.660664:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763097.660673:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763097.660675:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763097.660677:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763097.660684:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763097.660688:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763097.660690:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763097.660694:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763097.660698:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763097.660700:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763097.660702:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763097.660704:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763097.660706:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763097.660708:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763097.660713:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763097.660718:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763097.660724:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832590848/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763097.660733:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763097.660736:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763097.660740:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832590848/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763097.660751:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763097.660755:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763097.660760:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763097.660761:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763097.660763:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763097.660767:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763097.660770:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20812
00000400:00000200:0.0:1787763097.660774:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763097.660778:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763097.660781:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763097.660784:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763097.660787:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763097.660789:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763097.660791:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763097.660794:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763097.660796:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763097.660798:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763097.660800:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763097.660803:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763097.660805:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763097.660809:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763097.660811:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763097.660813:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763097.660815:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763097.660818:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763097.660821:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763097.660824:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763097.660826:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763097.660829:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763097.660831:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763097.660834:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763097.660837:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763097.660839:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763097.660841:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763097.660845:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763097.660849:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763097.660852:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763097.660854:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763097.660856:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763097.660858:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763097.660860:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763097.660863:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763097.660867:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763097.660871:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832590976/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763097.660880:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763097.660883:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763097.660887:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832590976/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.1:1787763097.660887:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763097.660892:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:0.0:1787763097.660897:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763097.660901:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763097.660905:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763097.660909:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763097.660913:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763097.660915:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763097.660917:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763097.660919:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763097.660922:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20812
00000400:00000200:0.0:1787763097.660926:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763097.660929:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763097.660931:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763097.660935:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763097.660938:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763097.660940:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763097.660943:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763097.660946:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763097.660948:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763097.660950:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763097.660953:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763097.660956:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763097.660958:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763097.660962:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763097.660964:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763097.660966:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763097.660968:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763097.660971:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763097.660974:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763097.660977:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763097.660979:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763097.660982:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763097.660984:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763097.660986:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763097.660990:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763097.660992:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763097.660994:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763097.660998:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763097.661000:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763097.661003:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763097.661005:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763097.661007:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763097.661008:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763097.661011:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763097.661015:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763097.661017:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763097.661022:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832591104/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763097.661040:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763097.661042:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763097.661046:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832591104/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763097.661056:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763097.661061:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20822)
00000800:00000200:3.1:1787763097.661826:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763097.661848:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787763097.662123:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763097.662125:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787763097.662403:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763097.662406:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:0.0:1787763099.324310:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763099.324316:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763099.324323:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763102.396476:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763102.396483:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763102.396487:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787763102.781282:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763102.781289:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787763102.782122:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763102.782125:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787763102.782560:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763102.782563:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787763102.782902:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763102.782906:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:0.0:1787763104.444313:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763104.444317:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763104.444319:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763105.468290:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763105.468293:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763105.468296:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763107.516418:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763107.516426:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763107.516433:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787763107.900789:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763107.900795:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787763107.901290:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763107.901292:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787763107.901506:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763107.901507:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787763107.901705:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763107.901707:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:0.0:1787763108.540430:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763108.540433:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763108.540436:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763110.588445:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763110.588451:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763110.588456:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763111.612383:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763111.612387:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763111.612390:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787763113.020734:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763113.020738:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787763113.021216:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763113.021230:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787763113.021447:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763113.021449:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787763113.021624:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763113.021625:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:0.0:1787763113.660352:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763113.660356:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763113.660358:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763115.708386:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763115.708396:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763115.708402:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763116.732272:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763116.732277:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763116.732280:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763118.140293:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763118.140333:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763118.140340:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763118.140348:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763118.140365:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763118.140384:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 20832
00000400:00000200:0.0:1787763118.140403:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763118.140416:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763118.140424:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763118.140431:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763118.140437:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763118.140443:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763118.140447:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763118.140454:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763118.140458:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763118.140463:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763118.140468:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763118.140471:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763118.140474:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763118.140479:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763118.140482:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763118.140485:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763118.140488:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763118.140492:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763118.140496:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763118.140500:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763118.140505:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763118.140510:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763118.140514:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763118.140517:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763118.140520:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763118.140531:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
00000020:00000040:0.0:1787763118.140536:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763118.140543:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763118.140545:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763118.140549:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763118.140552:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763118.140558:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763118.140568:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787763118.140576:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763118.140588:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832592768/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763118.140602:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763118.140608:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763118.140614:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832592768/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763118.140645:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763118.140651:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763118.140655:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763118.140657:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763118.140660:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763118.140664:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763118.140667:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20832
00000400:00000200:0.0:1787763118.140673:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763118.140676:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763118.140680:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763118.140683:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763118.140686:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763118.140689:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763118.140692:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763118.140695:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763118.140696:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763118.140699:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763118.140701:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763118.140704:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763118.140706:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763118.140710:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763118.140713:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763118.140716:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763118.140718:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763118.140722:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763118.140726:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763118.140729:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763118.140732:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763118.140739:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763118.140743:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763118.140745:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763118.140751:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763118.140758:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763118.140766:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763118.140776:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000020:00000040:0.0:1787763118.140785:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763118.140789:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763118.140795:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763118.140803:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763118.140808:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763118.140813:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763118.140824:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763118.140846:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763118.140861:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832592896/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763118.140889:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763118.140895:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763118.140910:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832592896/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763118.140929:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763118.140938:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763118.140950:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763118.140952:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763118.140955:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763118.140964:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763118.140973:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20832
00000400:00000200:0.0:1787763118.140980:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763118.140990:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763118.140995:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763118.141000:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763118.141004:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763118.141012:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763118.141016:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763118.141021:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763118.141026:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763118.141031:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763118.141037:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763118.141053:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763118.141065:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763118.141085:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763118.141097:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763118.141110:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763118.141121:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763118.141186:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763118.141201:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763118.141212:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763118.141221:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763118.141231:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763118.141240:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763118.141248:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763118.141253:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763118.141259:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763118.141265:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763118.141272:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000020:00000040:0.0:1787763118.141281:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763118.141288:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763118.141292:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763118.141300:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763118.141326:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763118.141330:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763118.141342:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763118.141352:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763118.141371:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832593024/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763118.141398:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763118.141409:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763118.141429:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763118.141431:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832593024/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.1:1787763118.141435:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:0.0:1787763118.141466:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763118.141473:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763118.141486:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763118.141495:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763118.141503:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763118.141511:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763118.141520:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763118.141527:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763118.141540:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20832
00000400:00000200:0.0:1787763118.141557:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763118.141571:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763118.141577:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763118.141590:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763118.141600:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763118.141609:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763118.141617:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763118.141627:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763118.141633:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763118.141642:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763118.141648:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763118.141652:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763118.141659:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763118.141670:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763118.141676:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763118.141684:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763118.141690:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763118.141701:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763118.141708:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763118.141718:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763118.141721:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763118.141726:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763118.141737:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763118.141741:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763118.141749:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763118.141757:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763118.141763:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763118.141774:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763118.141784:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763118.141794:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763118.141797:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763118.141805:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763118.141810:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763118.141817:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763118.141829:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763118.141843:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763118.141859:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832593152/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763118.141897:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763118.141905:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763118.141928:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832593152/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763118.141974:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763118.141988:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20842)
00000800:00000200:3.1:1787763118.142429:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763118.142432:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787763118.142804:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763118.142807:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787763118.143235:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763118.143240:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000020:00000001:0.0:1787763118.780527:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763118.780532:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763118.780537:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763119.804349:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763119.804355:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763119.804360:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763121.852446:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763121.852461:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763121.852465:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763122.876308:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763122.876312:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763122.876315:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787763123.261245:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763123.261252:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787763123.261579:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763123.261581:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787763123.261783:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763123.261785:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787763123.261991:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763123.261993:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000020:00000001:0.0:1787763124.924398:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763124.924413:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763124.924416:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763125.948428:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763125.948437:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763125.948444:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763127.996355:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763127.996361:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763127.996368:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787763128.381000:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763128.381006:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787763128.381722:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763128.381745:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787763128.382089:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763128.382091:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787763128.382385:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763128.382387:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:0.0:1787763129.020527:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763129.020536:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763129.020543:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763131.068364:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763131.068369:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763131.068375:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763132.092329:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763132.092336:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763132.092342:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763133.500264:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763133.500281:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763133.500287:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763133.500294:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763133.500302:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763133.500316:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 20848
00000400:00000200:0.0:1787763133.500337:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763133.500349:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763133.500362:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763133.500372:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763133.500377:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763133.500381:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763133.500384:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763133.500390:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763133.500393:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763133.500397:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763133.500402:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763133.500407:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763133.500410:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763133.500416:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763133.500419:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763133.500421:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763133.500424:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763133.500428:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763133.500437:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763133.500440:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763133.500445:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763133.500450:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763133.500454:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763133.500457:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763133.500461:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763133.500477:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763133.500483:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763133.500489:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763133.500491:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763133.500495:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763133.500499:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763133.500506:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763133.500518:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763133.500529:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763133.500543:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832594304/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763133.500557:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763133.500563:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763133.500569:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832594304/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763133.500597:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763133.500601:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763133.500605:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763133.500607:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763133.500610:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763133.500614:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763133.500617:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20848
00000400:00000200:0.0:1787763133.500623:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763133.500627:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763133.500631:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763133.500635:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763133.500638:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763133.500640:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763133.500643:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763133.500647:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763133.500651:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763133.500654:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763133.500660:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763133.500666:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763133.500669:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763133.500677:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763133.500683:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763133.500688:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763133.500694:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763133.500698:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763133.500704:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763133.500710:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763133.500714:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763133.500724:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763133.500730:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763133.500734:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763133.500742:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763133.500749:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763133.500752:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763133.500759:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763133.500765:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763133.500770:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763133.500773:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763133.500775:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763133.500778:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763133.500781:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763133.500789:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763133.500796:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763133.500807:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832594432/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763133.500818:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763133.500822:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763133.500831:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832594432/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763133.500844:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763133.500851:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763133.500856:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763133.500858:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763133.500860:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763133.500863:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763133.500867:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20848
00000400:00000200:0.0:1787763133.500872:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763133.500876:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763133.500878:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763133.500882:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763133.500884:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763133.500887:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763133.500890:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763133.500893:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763133.500895:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763133.500898:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763133.500900:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763133.500902:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763133.500905:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763133.500908:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763133.500911:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763133.500913:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763133.500915:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763133.500917:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763133.500920:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763133.500924:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763133.500926:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763133.500929:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763133.500931:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763133.500936:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763133.500938:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763133.500941:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763133.500943:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763133.500947:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763133.500950:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763133.500953:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763133.500954:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763133.500956:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763133.500958:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763133.500960:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763133.500964:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763133.500967:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763133.500972:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832594560/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763133.500980:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763133.500983:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763133.500987:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832594560/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763133.500996:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763133.500999:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763133.501004:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763133.501009:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763133.501013:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763133.501015:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763133.501017:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763133.501019:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763133.501023:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20848
00000400:00000200:0.0:1787763133.501027:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763133.501030:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763133.501033:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763133.501037:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763133.501040:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763133.501042:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763133.501046:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763133.501050:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763133.501052:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763133.501054:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763133.501059:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763133.501064:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763133.501070:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763133.501078:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763133.501083:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763133.501090:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763133.501096:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763133.501104:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763133.501108:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763133.501113:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763133.501115:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763133.501124:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763133.501127:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763133.501129:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763133.501182:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763133.501187:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763133.501190:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763133.501195:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763133.501215:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763133.501220:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763133.501224:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763133.501226:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763133.501229:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763133.501234:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763133.501243:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787763133.501249:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763133.501259:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832594688/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763133.501273:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763133.501277:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763133.501286:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832594688/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763133.501304:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763133.501312:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20858)
00000800:00000200:3.1:1787763133.501347:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763133.501352:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787763133.501807:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763133.501809:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787763133.502105:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763133.502107:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787763133.502388:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763133.502391:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:0.0:1787763134.140441:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763134.140448:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763134.140452:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763135.164372:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763135.164378:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763135.164382:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763136.188359:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763136.188365:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763136.188369:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763137.212505:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763137.212526:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763137.212534:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763138.236311:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763138.236315:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763138.236318:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763138.620324:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763138.620329:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763138.620332:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763138.620358:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763138.620366:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763138.620372:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763138.620379:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763138.620385:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763138.620393:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763138.621143:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763138.621151:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787763138.622370:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763138.622373:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787763138.622812:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763138.622815:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787763138.623228:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763138.623231:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:0.0:1787763139.260500:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763139.260507:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763139.260511:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763139.644275:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763139.644285:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763139.644288:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763139.644296:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763139.644303:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763139.644308:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763139.644311:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763139.644315:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763139.644318:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763140.668282:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763140.668295:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763140.668299:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763140.668306:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763140.668310:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763140.668315:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763140.668319:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763140.668322:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763140.668326:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763141.692251:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763141.692260:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763141.692262:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763141.692267:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763141.692271:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763141.692274:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763141.692277:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763141.692279:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763141.692281:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763142.332411:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763142.332417:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763142.332421:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763142.716248:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763142.716260:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763142.716263:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763142.716272:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763142.716278:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763142.716286:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763142.716290:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763142.716294:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763142.716299:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763143.740299:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763143.740305:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763143.740308:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763143.740317:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763143.740327:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763143.740332:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763143.740337:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763143.740341:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763143.740346:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763143.741118:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763143.741124:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787763143.742026:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763143.742044:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787763143.742402:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763143.742405:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787763143.742766:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763143.742769:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:0.0:1787763144.764216:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763144.764224:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763144.764227:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763144.764232:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763144.764237:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763144.764241:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763144.764245:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763144.764248:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763144.764251:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763145.404396:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763145.404402:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763145.404407:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763145.788256:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763145.788267:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763145.788270:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763145.788278:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763145.788285:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763145.788290:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763145.788295:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763145.788299:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763145.788302:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763146.812204:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763146.812210:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763146.812212:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763146.812217:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763146.812220:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763146.812223:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763146.812225:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763146.812227:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763146.812228:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763147.836258:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763147.836269:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763147.836273:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763147.836293:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763147.836299:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763147.836304:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763147.836308:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763147.836311:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763147.836314:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763148.476499:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763148.476505:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763148.476510:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763148.860241:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763148.860248:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763148.860251:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763148.860256:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763148.860261:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763148.860266:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763148.860268:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763148.860281:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763148.860288:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763148.860925:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763148.860932:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787763148.861845:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763148.861848:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787763148.862194:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763148.862196:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787763148.862588:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763148.862591:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000100:00000001:0.0:1787763149.884279:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763149.884291:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763149.884296:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763149.884306:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763149.884314:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763149.884320:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763149.884326:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763149.884331:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763149.884337:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000008:00000020:0.0:1787763150.396236:0:304413:0:(osc_request.c:747:osc_update_next_shrink()) next time 22070 to shrink grant
00000100:00000001:0.0:1787763150.908263:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763150.908274:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763150.908277:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763150.908286:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763150.908294:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763150.908298:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763150.908302:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763150.908321:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763150.908328:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763151.548359:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763151.548365:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763151.548370:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763151.932247:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763151.932260:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763151.932266:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763151.932276:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763151.932285:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763151.932292:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763151.932299:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763151.932305:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763151.932312:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763152.956237:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763152.956261:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763152.956264:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763152.956273:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763152.956279:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763152.956284:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763152.956288:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763152.956291:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763152.956295:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763153.980234:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763153.980250:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763153.980255:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763153.980260:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763153.980266:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763153.980271:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763153.980276:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763153.980280:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763153.980283:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763153.980844:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763153.980855:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787763153.981589:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763153.981591:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787763153.981874:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763153.981877:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787763153.982159:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763153.982162:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000020:00000001:0.0:1787763154.620311:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763154.620316:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763154.620319:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763155.004224:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763155.004232:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763155.004235:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763155.004243:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763155.004249:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763155.004253:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763155.004256:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763155.004260:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763155.004263:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763156.028255:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763156.028264:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763156.028266:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763156.028271:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763156.028274:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763156.028278:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763156.028281:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763156.028284:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763156.028287:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763157.052249:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763157.052260:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763157.052263:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763157.052271:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763157.052278:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763157.052285:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763157.052291:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763157.052296:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763157.052301:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763157.692455:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763157.692477:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763157.692484:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763158.076250:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763158.076261:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763158.076266:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763158.076275:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763158.076283:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763158.076306:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763158.076312:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763158.076317:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763158.076322:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763158.716328:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763158.716340:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763158.716345:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763158.716354:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763158.716362:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763158.716368:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763158.716371:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763158.716375:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763158.716378:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763159.100305:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763159.100316:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763159.100324:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763159.100340:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763159.100355:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763159.100363:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763159.100371:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763159.100375:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763159.100388:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763159.101290:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763159.102194:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763159.102527:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763159.102838:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763159.740254:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763159.740266:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763159.740271:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763159.740284:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763159.740298:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763159.740306:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763159.740313:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763159.740318:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763159.740323:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763160.124396:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763160.124406:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763160.124409:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763160.124415:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763160.124420:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763160.124425:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763160.124428:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763160.124431:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763160.124435:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763160.764214:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763160.764224:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763160.764226:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763160.764234:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763160.764239:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763160.764244:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763160.764248:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763160.764252:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763160.764256:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763161.148328:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763161.148341:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763161.148345:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763161.148353:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763161.148357:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763161.148361:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763161.148365:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763161.148368:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763161.148373:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763161.788190:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763161.788201:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763161.788204:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763161.788210:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763161.788214:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763161.788218:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763161.788221:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763161.788224:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763161.788228:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763162.172409:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763162.172417:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763162.172420:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763162.172425:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763162.172429:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763162.172433:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.172435:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763162.172438:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.172440:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763162.172993:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763162.173012:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763162.173016:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763162.173024:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763159/real 1787763162]  req@ffff9751f86583c0 x1874599832597120/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763162.173038:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763162.173041:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763162.173046:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc234840.
00000400:00000200:0.0:1787763162.173051:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32f68
00000400:00000010:0.0:1787763162.173052:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32f68.
00000100:00000001:0.0:1787763162.173056:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763162.173060:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832597120/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763162.173070:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832597120/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763162.173077:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763162.173086:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.173090:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763162.173093:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763162.173097:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832597120/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763162.173108:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763162.173114:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832597120/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763162.173126:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763162.173129:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.173131:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763162.173176:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.173181:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763162.173184:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763162.173189:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763162.173196:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763162.173199:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763162.173204:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832597120/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763162.173215:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832597120:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763162.173221:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763162.173225:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763159/real 1787763162]  req@ffff9751f8658780 x1874599832597248/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763162.173235:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763162.173236:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763162.173239:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc234478.
00000400:00000200:0.0:1787763162.173242:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32880
00000400:00000010:0.0:1787763162.173244:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32880.
00000100:00000001:0.0:1787763162.173247:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763162.173251:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832597248/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763162.173259:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832597248/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763162.173266:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763162.173267:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.173270:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763162.173271:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763162.173274:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832597248/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763162.173281:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763162.173285:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832597248/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763162.173292:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763162.173294:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.173296:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763162.173297:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.173299:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763162.173301:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763162.173304:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763162.173308:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763162.173311:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763162.173314:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832597248/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763162.173323:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832597248:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763162.173328:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763162.173333:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763159/real 1787763162]  req@ffff9751f8659a40 x1874599832596992/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763162.173341:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763162.173343:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763162.173345:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc234aa8.
00000400:00000200:0.0:1787763162.173348:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32b28
00000400:00000010:0.0:1787763162.173350:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32b28.
00000100:00000001:0.0:1787763162.173353:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763162.173356:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832596992/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763162.173366:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832596992/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763162.173373:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763162.173375:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.173377:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763162.173378:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763162.173382:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832596992/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763162.173389:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763162.173394:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832596992/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763162.173402:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763162.173405:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.173406:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763162.173408:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.173410:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763162.173427:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763162.173430:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763162.173434:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763162.173438:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763162.173442:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832596992/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763162.173450:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832596992:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763162.173455:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763162.173459:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763159/real 1787763162]  req@ffff9751f8658000 x1874599832596864/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763162.173468:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763162.173470:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763162.173472:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc234580.
00000400:00000200:0.0:1787763162.173475:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d324c8
00000400:00000010:0.0:1787763162.173478:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d324c8.
00000100:00000001:0.0:1787763162.173481:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763162.173484:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832596864/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763162.173494:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832596864/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763162.173501:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763162.173503:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.173505:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763162.173506:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763162.173510:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832596864/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763162.173518:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763162.173523:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832596864/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763162.173530:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763162.173533:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.173536:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763162.173538:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.173539:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763162.173542:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763162.173545:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763162.173549:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763162.173552:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763162.173556:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832596864/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763162.173565:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832596864:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763162.173570:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.173573:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763162.173576:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832597120/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763162.173584:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763162.173588:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763162.173593:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611400.
02000000:00000001:0.0:1787763162.173598:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763162.173602:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763162.173605:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763162.173608:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763162.173611:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:0.0:1787763162.173617:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763162.173619:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763162.173622:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:0.0:1787763162.173625:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763162.173627:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.173628:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763162.173632:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832597248/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763162.173640:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763162.173642:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763162.173644:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610000.
02000000:00000001:0.0:1787763162.173647:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763162.173648:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763162.173650:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787763162.173652:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763162.173663:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:0.0:1787763162.173666:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763162.173668:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763162.173670:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:0.0:1787763162.173673:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763162.173675:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.173677:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763162.173681:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832596992/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763162.173687:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763162.173688:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763162.173690:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613400.
02000000:00000001:0.0:1787763162.173692:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763162.173693:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763162.173694:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763162.173695:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763162.173696:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:0.0:1787763162.173698:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763162.173699:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763162.173701:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:0.0:1787763162.173703:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763162.173704:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.173705:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763162.173708:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832596864/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763184 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763162.173712:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763162.173713:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763162.173713:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611000.
02000000:00000001:0.0:1787763162.173715:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763162.173715:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763162.173717:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787763162.173719:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763162.173720:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:0.0:1787763162.173722:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763162.173724:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763162.173725:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:0.0:1787763162.173727:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763162.173727:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.173730:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.173739:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763162.173740:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763162.173742:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763162.173743:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763162.812206:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763162.812214:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763162.812217:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763162.812223:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763162.812229:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763162.812234:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.812237:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763162.812240:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763162.812243:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763163.196314:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763163.196325:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763163.196328:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763163.196343:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763163.196348:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763163.196353:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763163.196356:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763163.196359:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763163.196362:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763163.836242:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763163.836253:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763163.836257:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763163.836263:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763163.836270:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763163.836276:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763163.836282:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763163.836288:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763163.836296:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763164.220305:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763164.220319:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763164.220332:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763164.220347:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763164.220359:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763164.220368:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.220377:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763164.220380:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.220384:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763164.220704:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763164.220718:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763164.220727:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763164.220750:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832597376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.220792:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763164.220800:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763164.220812:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763164.220822:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763164.220837:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763164.220855:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832597376:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763164.220871:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763164.220882:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763164.220891:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763164.220903:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763164.220917:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763164.220929:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763164.220948:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763164.220960:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763164.220980:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763164.221025:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832597376, portal 25
00000100:00000001:0.0:1787763164.221038:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763164.221043:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:0.0:1787763164.221060:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832597376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.221092:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763164.221106:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763164.221115:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763164.221130:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832597376, offset 0
00000400:00000200:0.0:1787763164.221195:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763164.221211:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763164.221226:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763164.221240:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763164.221252:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763164.221268:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763164.221281:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763164.221297:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57902:57902:256:4294967295] 192.168.203.110@tcp LPNI seq info [57902:57902:8:4294967295]
00000400:00000200:0.0:1787763164.221329:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763164.221366:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763164.221392:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763164.221409:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763164.221429:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763164.221444:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787763164.221471:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763164.221484:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787763164.221507:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763164.221517:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763164.221532:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763164.221554:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763164.221567:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763164.221592:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763164.221608:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763164.221625:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832597376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.221673:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763164.221690:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832597376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763189 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.221746:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763164.221754:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763164.221772:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763164.221790:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763164.221801:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315552500
00000100:00000010:0.0:1787763164.221819:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c420.
00000100:00000001:0.0:1787763164.221838:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763164.221857:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315552500
00000100:00000010:0.0:1787763164.221874:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940cd80.
00000100:00000001:0.0:1787763164.221886:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787763164.221894:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763164.221901:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763164.221909:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763164.221924:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763164/real 1787763164]  req@ffff9751f865ad00 x1874599832597376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763164.221956:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763164.221966:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763164.221977:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763164.222021:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763164.222032:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763164.222045:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763164.222062:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832597376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763164.222106:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832597376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.222190:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763164.222200:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.222211:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763164.222221:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763164.222240:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832597376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.222283:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763164.222304:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832597376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.222335:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763164.222344:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.222348:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763164.222353:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.222359:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763164.222363:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763164.222370:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763164.222383:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763164.222392:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763164.222407:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832597376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763164.222440:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832597376:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763164.222453:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.222459:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763164.222470:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832597376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.222480:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763164.222483:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763164.222487:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763164.222492:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763164.222495:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763164.222498:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787763164.222501:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763164.222504:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:0.0:1787763164.222508:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763164.222511:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763164.222514:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:0.0:1787763164.222518:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763164.222519:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.222522:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.222531:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763164.222534:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763164.222537:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763164.222539:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763164.222541:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763164.222546:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832597504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.222555:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763164.222557:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763164.222561:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763164.222563:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763164.222566:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763164.222570:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832597504:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763164.222574:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763164.222578:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763164.222581:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763164.222585:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763164.222592:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763164.222596:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763164.222605:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763164.222610:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763164.222617:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763164.222624:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832597504, portal 10
00000100:00000001:0.0:1787763164.222628:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763164.222631:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:0.0:1787763164.222644:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832597504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.222659:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763164.222664:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763164.222670:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763164.222676:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832597504, offset 0
00000400:00000200:0.0:1787763164.222685:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763164.222693:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763164.222699:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763164.222704:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763164.222708:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763164.222712:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763164.222715:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763164.222719:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57903:57903:256:4294967295] 192.168.203.110@tcp LPNI seq info [57903:57903:8:4294967295]
00000400:00000200:0.0:1787763164.222725:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763164.222737:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763164.222746:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763164.222753:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763164.222762:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763164.222768:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787763164.222777:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763164.222783:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787763164.222792:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763164.222796:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763164.222803:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763164.222811:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763164.222818:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763164.222822:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763164.222828:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763164.222835:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832597504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.222846:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763164.222850:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832597504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763189 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.222858:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763164.222860:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763164.222867:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763164.222873:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:0.0:1787763164.222877:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763164.222880:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315551265
00000100:00000001:0.0:1787763164.222884:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315552500
00000100:00000001:0.0:1787763164.222887:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315551265
00000100:00000001:0.0:1787763164.222890:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315552500
00000100:00000001:0.0:1787763164.222892:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763164.222895:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763164.222898:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763164.222902:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763164/real 1787763164]  req@ffff9751f8658000 x1874599832597504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763164.222913:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763164.222915:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763164.222917:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763164.222921:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763164.222923:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763164.222926:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763164.222930:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832597504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763164.222941:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832597504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.222949:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763164.222951:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.222953:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763164.222955:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763164.222959:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832597504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.222967:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763164.222972:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832597504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.222981:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763164.222984:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.223016:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763164.223019:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.223022:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763164.223037:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763164.223040:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763164.223045:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763164.223048:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763164.223054:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832597504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763164.223065:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832597504:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763164.223070:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763164.223074:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832597632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.223082:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763164.223084:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763164.223087:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763164.223089:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763164.223092:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763164.223095:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832597632:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763164.223100:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763164.223103:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763164.223105:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763164.223107:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763164.223113:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015400.
02000000:00000001:0.0:1787763164.223115:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763164.223120:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763164.223124:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763164.223128:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763164.223159:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832597632, portal 10
00000100:00000001:0.0:1787763164.223163:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763164.223165:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:0.0:1787763164.223173:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832597632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.223182:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763164.223185:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763164.223188:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763164.223192:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832597632, offset 0
00000400:00000200:0.0:1787763164.223197:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763164.223203:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763164.223208:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763164.223212:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763164.223216:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763164.223220:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763164.223223:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763164.223227:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57904:57904:256:4294967295] 192.168.203.110@tcp LPNI seq info [57904:57904:8:4294967295]
00000400:00000200:0.0:1787763164.223232:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763164.223241:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763164.223247:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763164.223251:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763164.223256:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763164.223259:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787763164.223265:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763164.223269:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787763164.223274:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763164.223276:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763164.223280:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763164.223285:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763164.223289:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763164.223291:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763164.223294:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763164.223299:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832597632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.223309:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763164.223313:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832597632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763189 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.223322:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763164.223323:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763164.223328:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763164.223331:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:0.0:1787763164.223335:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763164.223338:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315551254
00000100:00000001:0.0:1787763164.223341:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315552500
00000100:00000001:0.0:1787763164.223344:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315551254
00000100:00000001:0.0:1787763164.223346:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315552500
00000100:00000001:0.0:1787763164.223349:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763164.223352:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763164.223354:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763164.223358:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763164/real 1787763164]  req@ffff9751f8659a40 x1874599832597632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763164.223367:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763164.223369:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763164.223371:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763164.223374:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763164.223376:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763164.223379:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763164.223383:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832597632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763164.223393:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832597632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.223401:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763164.223403:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.223406:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763164.223407:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763164.223412:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832597632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.223420:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763164.223424:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832597632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.223432:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763164.223435:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.223437:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763164.223439:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.223441:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763164.223443:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763164.223446:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763164.223449:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763164.223452:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763164.223456:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832597632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763164.223466:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832597632:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763164.223470:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763164.223474:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832597760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.223484:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763164.223486:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763164.223490:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763164.223493:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763164.223496:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763164.223501:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832597760:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763164.223506:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763164.223509:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763164.223511:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763164.223513:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763164.223518:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016000.
02000000:00000001:0.0:1787763164.223521:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763164.223525:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763164.223528:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763164.223532:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763164.223536:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832597760, portal 4
00000100:00000001:0.0:1787763164.223538:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763164.223540:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000040:0.0:1787763164.223547:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832597760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.223555:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763164.223558:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763164.223561:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763164.223563:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832597760, offset 0
00000400:00000200:0.0:1787763164.223568:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763164.223572:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763164.223576:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763164.223580:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763164.223585:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763164.223590:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763164.223595:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763164.223601:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57905:57905:256:4294967295] 192.168.203.110@tcp LPNI seq info [57905:57905:8:4294967295]
00000400:00000200:0.0:1787763164.223610:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763164.223623:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763164.223632:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763164.223639:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763164.223647:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763164.223653:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787763164.223662:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763164.223669:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787763164.223677:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763164.223682:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763164.223688:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763164.223694:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763164.223698:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763164.223701:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763164.223704:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763164.223709:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832597760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763189 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.223721:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763164.223725:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832597760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763189 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.223737:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763164.223741:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763164.223749:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763164.223755:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:0.0:1787763164.223763:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763164.223767:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315551271
00000100:00000001:0.0:1787763164.223772:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315552500
00000100:00000001:0.0:1787763164.223777:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315551271
00000100:00000001:0.0:1787763164.223781:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315552500
00000100:00000001:0.0:1787763164.223786:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763164.223790:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763164.223795:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763164.223803:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763164/real 1787763164]  req@ffff9751f8658780 x1874599832597760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763164.223823:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763164.223826:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763164.223830:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763164.223834:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763164.223836:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763164.223839:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763164.223844:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832597760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763164.223855:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832597760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.223863:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763164.223865:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.223867:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763164.223869:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763164.223873:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832597760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.223881:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763164.223886:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832597760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.223895:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763164.223898:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.223900:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763164.223902:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.223904:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763164.223906:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763164.223909:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763164.223913:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763164.223915:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763164.223921:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832597760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763164.223930:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832597760:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763164.223935:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.223938:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763164.223942:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832597504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.223950:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763164.223952:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763164.223954:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763164.223957:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763164.223959:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763164.223961:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763164.223964:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763164.223966:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:0.0:1787763164.223970:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763164.223972:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763164.223974:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:0.0:1787763164.223977:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763164.223978:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.223980:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763164.223984:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832597632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.224016:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763164.224018:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763164.224020:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015400.
02000000:00000001:0.0:1787763164.224023:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763164.224025:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763164.224026:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763164.224029:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763164.224031:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:0.0:1787763164.224034:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763164.224036:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763164.224038:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:0.0:1787763164.224041:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763164.224042:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.224044:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763164.224049:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832597760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763189 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763164.224057:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763164.224058:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763164.224060:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016000.
02000000:00000001:0.0:1787763164.224063:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763164.224065:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763164.224066:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787763164.224068:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763164.224070:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:0.0:1787763164.224072:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763164.224073:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763164.224075:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:0.0:1787763164.224077:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763164.224079:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.224081:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.224086:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763164.224088:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763164.224090:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763164.224093:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763164.860244:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763164.860253:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763164.860256:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763164.860261:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763164.860265:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763164.860269:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.860276:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763164.860279:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763164.860284:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763165.244363:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763165.244372:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763165.244374:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763165.244380:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763165.244384:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763165.244388:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763165.244390:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763165.244393:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763165.244396:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763165.884293:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763165.884303:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763165.884319:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763165.884344:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763165.884358:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763165.884364:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763165.884386:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763165.884392:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763165.884396:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763166.908221:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763166.908227:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763166.908229:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763166.908234:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763166.908238:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763166.908242:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763166.908250:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763166.908255:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763166.908261:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763167.932245:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763167.932255:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763167.932259:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763167.932267:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763167.932274:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763167.932280:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763167.932284:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763167.932288:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763167.932292:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763168.956193:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763168.956200:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763168.956202:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763168.956208:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763168.956213:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763168.956216:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763168.956221:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763168.956223:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763168.956227:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763169.340660:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763169.340672:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763169.340679:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763169.340697:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832597888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.340736:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763169.340741:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763169.340750:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763169.340758:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763169.340765:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763169.340778:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832597888:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763169.340794:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763169.340802:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763169.340808:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763169.340816:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763169.340828:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763169.340835:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763169.340850:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763169.340859:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763169.340872:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763169.340883:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832597888, portal 25
00000100:00000001:0.0:1787763169.340890:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763169.340896:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:0.0:1787763169.340912:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832597888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.340932:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763169.340939:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763169.340977:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763169.340990:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832597888, offset 0
00000400:00000200:0.0:1787763169.341011:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763169.341029:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763169.341048:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763169.341057:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763169.341083:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763169.341103:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763169.341118:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763169.341194:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57906:57906:256:4294967295] 192.168.203.110@tcp LPNI seq info [57906:57906:8:4294967295]
00000400:00000200:0.0:1787763169.341216:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763169.341251:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763169.341272:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763169.341284:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763169.341305:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763169.341313:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1787763169.341333:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763169.341346:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:0.0:1787763169.341360:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763169.341365:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763169.341378:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763169.341396:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763169.341407:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763169.341413:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763169.341420:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763169.341431:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832597888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.341475:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763169.341488:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832597888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763194 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.341509:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763169.341512:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763169.341523:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763169.341538:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763169.341548:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315557620
00000100:00000010:0.0:1787763169.341562:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940cb40.
00000100:00000001:0.0:1787763169.341581:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763169.341599:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315557620
00000100:00000010:0.0:1787763169.341613:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c3c0.
00000100:00000001:0.0:1787763169.341626:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787763169.341636:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763169.341652:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763169.341661:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763169.341674:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763169/real 1787763169]  req@ffff9751f86583c0 x1874599832597888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763169.341708:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763169.341715:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763169.341720:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763169.341732:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763169.341741:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763169.341754:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763169.341771:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832597888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763169.341813:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832597888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.341837:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763169.341845:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.341856:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763169.341864:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763169.341874:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832597888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.341898:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763169.341912:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832597888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.341940:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763169.341947:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.341978:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763169.341983:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.341993:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763169.341999:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763169.342006:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763169.342024:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763169.342032:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763169.342046:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832597888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763169.342094:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832597888:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763169.342111:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.342121:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763169.342128:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832597888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.342166:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763169.342169:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763169.342174:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763169.342181:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763169.342183:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763169.342185:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787763169.342189:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763169.342192:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:0.0:1787763169.342197:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763169.342199:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763169.342201:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:0.0:1787763169.342205:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763169.342206:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.342209:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.342218:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763169.342220:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763169.342223:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763169.342225:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763169.342227:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763169.342233:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832598016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.342242:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763169.342244:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763169.342247:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763169.342250:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763169.342253:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763169.342257:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832598016:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763169.342261:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763169.342264:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763169.342266:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763169.342269:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763169.342273:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763169.342276:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763169.342281:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763169.342285:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763169.342289:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763169.342294:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832598016, portal 10
00000100:00000001:0.0:1787763169.342296:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763169.342298:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000040:0.0:1787763169.342307:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832598016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.342315:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763169.342318:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763169.342321:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763169.342324:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832598016, offset 0
00000400:00000200:0.0:1787763169.342329:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763169.342335:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763169.342340:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763169.342344:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763169.342348:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763169.342351:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763169.342354:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763169.342359:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57907:57907:256:4294967295] 192.168.203.110@tcp LPNI seq info [57907:57907:8:4294967295]
00000400:00000200:0.0:1787763169.342364:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763169.342373:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763169.342378:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763169.342382:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763169.342388:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763169.342392:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1787763169.342398:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763169.342401:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:0.0:1787763169.342406:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763169.342408:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763169.342412:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763169.342417:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763169.342421:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763169.342423:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763169.342427:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763169.342431:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832598016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.342441:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763169.342445:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832598016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763194 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.342453:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763169.342456:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763169.342461:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763169.342465:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:0.0:1787763169.342469:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763169.342472:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315556385
00000100:00000001:0.0:1787763169.342476:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315557620
00000100:00000001:0.0:1787763169.342479:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315556385
00000100:00000001:0.0:1787763169.342482:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315557620
00000100:00000001:0.0:1787763169.342485:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763169.342488:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763169.342490:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763169.342495:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763169/real 1787763169]  req@ffff9751f8658780 x1874599832598016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763169.342505:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763169.342506:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763169.342508:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763169.342512:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763169.342514:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763169.342516:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763169.342520:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832598016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763169.342530:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832598016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.342539:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763169.342540:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.342543:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763169.342546:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763169.342553:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832598016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.342567:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763169.342575:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832598016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.342589:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763169.342594:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.342597:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763169.342600:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.342603:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763169.342622:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763169.342628:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763169.342635:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763169.342641:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763169.342649:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832598016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763169.342663:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832598016:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763169.342668:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763169.342672:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832598144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.342681:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763169.342683:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763169.342686:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763169.342688:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763169.342691:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763169.342694:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832598144:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763169.342698:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763169.342701:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763169.342703:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763169.342705:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763169.342710:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016000.
02000000:00000001:0.0:1787763169.342713:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763169.342718:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763169.342721:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763169.342725:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763169.342729:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832598144, portal 10
00000100:00000001:0.0:1787763169.342732:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763169.342734:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:0.0:1787763169.342741:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832598144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.342757:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763169.342760:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763169.342762:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763169.342766:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832598144, offset 0
00000400:00000200:0.0:1787763169.342771:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763169.342776:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763169.342780:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763169.342784:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763169.342788:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763169.342791:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763169.342794:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763169.342799:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57908:57908:256:4294967295] 192.168.203.110@tcp LPNI seq info [57908:57908:8:4294967295]
00000400:00000200:0.0:1787763169.342804:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763169.342812:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763169.342818:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763169.342822:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763169.342827:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763169.342831:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1787763169.342836:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763169.342839:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:0.0:1787763169.342844:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763169.342847:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763169.342851:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763169.342855:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763169.342859:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763169.342861:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763169.342864:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763169.342869:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832598144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.342879:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763169.342882:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832598144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763194 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.342899:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763169.342901:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763169.342906:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763169.342909:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:0.0:1787763169.342913:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763169.342916:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315556373
00000100:00000001:0.0:1787763169.342919:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315557620
00000100:00000001:0.0:1787763169.342922:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315556373
00000100:00000001:0.0:1787763169.342925:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315557620
00000100:00000001:0.0:1787763169.342927:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763169.342930:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763169.342932:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763169.342936:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763169/real 1787763169]  req@ffff9751f8659a40 x1874599832598144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763169.342945:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763169.342947:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763169.342949:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763169.342977:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763169.342979:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763169.342983:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763169.342987:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832598144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763169.342998:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832598144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.343006:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763169.343008:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.343011:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763169.343012:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763169.343016:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832598144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.343026:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763169.343031:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832598144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.343039:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763169.343042:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.343044:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763169.343046:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.343048:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763169.343050:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763169.343053:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763169.343057:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763169.343059:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763169.343064:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832598144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763169.343073:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832598144:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763169.343078:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763169.343081:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832598272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.343090:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763169.343092:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763169.343095:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763169.343097:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763169.343100:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763169.343103:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832598272:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763169.343107:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763169.343109:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763169.343112:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763169.343114:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763169.343119:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015400.
02000000:00000001:0.0:1787763169.343121:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763169.343126:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763169.343129:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763169.343132:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763169.343163:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832598272, portal 4
00000100:00000001:0.0:1787763169.343166:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763169.343167:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:0.0:1787763169.343175:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832598272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.343183:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763169.343186:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763169.343190:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763169.343193:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832598272, offset 0
00000400:00000200:0.0:1787763169.343197:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763169.343203:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763169.343208:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763169.343212:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763169.343215:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763169.343218:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763169.343221:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763169.343225:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57909:57909:256:4294967295] 192.168.203.110@tcp LPNI seq info [57909:57909:8:4294967295]
00000400:00000200:0.0:1787763169.343231:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763169.343239:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763169.343244:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763169.343248:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763169.343253:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763169.343256:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:0.1:1787763169.343261:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763169.343265:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:0.0:1787763169.343269:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763169.343273:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763169.343276:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763169.343280:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763169.343284:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763169.343286:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763169.343289:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763169.343293:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832598272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763194 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.343303:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763169.343307:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832598272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763194 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.343316:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763169.343318:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763169.343322:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763169.343326:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:0.0:1787763169.343329:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763169.343332:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315556391
00000100:00000001:0.0:1787763169.343335:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315557620
00000100:00000001:0.0:1787763169.343338:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315556391
00000100:00000001:0.0:1787763169.343340:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315557620
00000100:00000001:0.0:1787763169.343343:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763169.343345:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763169.343348:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763169.343352:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763169/real 1787763169]  req@ffff9751f8658000 x1874599832598272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763169.343361:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763169.343363:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763169.343366:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763169.343369:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763169.343371:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763169.343374:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763169.343378:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832598272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763169.343388:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832598272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.343396:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763169.343397:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.343400:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763169.343401:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763169.343405:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832598272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.343413:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763169.343418:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832598272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.343425:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763169.343428:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.343430:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763169.343431:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.343434:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763169.343436:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763169.343439:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763169.343443:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763169.343445:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763169.343450:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832598272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763169.343460:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832598272:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763169.343465:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.343467:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763169.343471:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832598016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.343480:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763169.343482:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763169.343484:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763169.343488:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763169.343489:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763169.343491:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763169.343493:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763169.343496:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:0.0:1787763169.343499:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763169.343501:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763169.343503:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:0.0:1787763169.343506:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763169.343508:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.343510:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763169.343514:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832598144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.343522:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763169.343524:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763169.343525:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016000.
02000000:00000001:0.0:1787763169.343528:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763169.343529:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763169.343531:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763169.343533:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763169.343534:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:0.0:1787763169.343537:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763169.343538:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763169.343540:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:0.0:1787763169.343543:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763169.343546:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.343549:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763169.343556:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832598272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763194 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763169.343569:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763169.343572:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763169.343576:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015400.
02000000:00000001:0.0:1787763169.343580:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763169.343583:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763169.343585:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787763169.343589:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763169.343592:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:0.0:1787763169.343596:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763169.343599:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763169.343602:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:0.0:1787763169.343606:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763169.343608:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.343612:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.343619:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763169.343623:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763169.343627:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763169.343630:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763169.980314:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763169.980326:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763169.980333:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763169.980342:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763169.980353:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763169.980360:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.980367:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763169.980372:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763169.980376:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763171.004327:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763171.004336:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763171.004338:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763171.004344:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763171.004350:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763171.004354:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763171.004358:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763171.004361:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763171.004365:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763172.028315:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763172.028342:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763172.028347:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763172.028364:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763172.028375:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763172.028380:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763172.028384:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763172.028387:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763172.028391:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763174.460720:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763174.460732:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763174.460744:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763174.460761:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832598400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.460804:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763174.460812:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763174.460840:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763174.460849:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763174.460867:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763174.460893:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832598400:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763174.460913:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763174.460934:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763174.460938:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763174.460943:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763174.460958:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016000.
02000000:00000001:0.0:1787763174.460968:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763174.460985:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763174.460994:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763174.461007:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763174.461023:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832598400, portal 25
00000100:00000001:0.0:1787763174.461032:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763174.461038:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:0.0:1787763174.461048:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832598400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763199 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.461121:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763174.461182:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763174.461187:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763174.461198:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832598400, offset 0
00000400:00000200:0.0:1787763174.461222:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763174.461257:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763174.461270:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763174.461281:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763174.461298:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763174.461310:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763174.461328:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763174.461349:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57910:57910:256:4294967295] 192.168.203.110@tcp LPNI seq info [57910:57910:8:4294967295]
00000400:00000200:0.0:1787763174.461361:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763174.461411:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763174.461431:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763174.461447:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763174.461466:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763174.461474:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787763174.461499:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763174.461509:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:0.0:1787763174.461540:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763174.461546:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763174.461560:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763174.461575:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763174.461583:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763174.461590:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763174.461607:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763174.461628:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832598400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763199 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.461650:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763174.461661:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832598400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763199 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.461739:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763174.461749:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763174.461763:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763174.461784:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763174.461790:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315562740
00000100:00000010:0.0:1787763174.461800:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c840.
00000100:00000001:0.0:1787763174.461812:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763174.461840:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315562740
00000100:00000010:0.0:1787763174.461855:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c6c0.
00000100:00000001:0.0:1787763174.461868:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787763174.461881:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763174.461892:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763174.461907:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763174.461936:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763174/real 1787763174]  req@ffff9751f865ad00 x1874599832598400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763174.461970:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763174.461977:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763174.461983:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763174.461997:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763174.462006:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763174.462020:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763174.462034:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832598400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763174.462054:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832598400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.462077:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763174.462087:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763174.462101:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763174.462111:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763174.462128:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832598400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.462166:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763174.462179:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832598400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.462254:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763174.462265:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763174.462276:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763174.462283:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763174.462294:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763174.462298:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763174.462305:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763174.462323:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763174.462328:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763174.462333:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832598400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763174.462354:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832598400:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763174.462359:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763174.462362:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763174.462364:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832598400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.462368:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763174.462371:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763174.462374:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016000.
02000000:00000001:0.0:1787763174.462378:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763174.462381:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763174.462382:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787763174.462384:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763174.462387:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:0.0:1787763174.462391:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763174.462393:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763174.462394:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:0.0:1787763174.462397:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763174.462398:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763174.462400:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763174.462408:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763174.462410:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763174.462411:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763174.462413:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763174.462415:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763174.462418:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832598528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.462423:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763174.462425:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763174.462427:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763174.462429:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763174.462430:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763174.462432:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832598528:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763174.462435:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763174.462437:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763174.462439:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763174.462440:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763174.462443:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016000.
02000000:00000001:0.0:1787763174.462444:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763174.462448:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763174.462451:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763174.462454:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763174.462457:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832598528, portal 10
00000100:00000001:0.0:1787763174.462459:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763174.462460:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:0.0:1787763174.462466:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832598528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763199 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.462471:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763174.462472:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763174.462474:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763174.462476:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832598528, offset 0
00000400:00000200:0.0:1787763174.462479:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763174.462482:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763174.462487:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763174.462489:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763174.462491:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763174.462493:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763174.462495:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763174.462498:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57911:57911:256:4294967295] 192.168.203.110@tcp LPNI seq info [57911:57911:8:4294967295]
00000400:00000200:0.0:1787763174.462501:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763174.462506:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763174.462509:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763174.462512:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763174.462515:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763174.462517:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787763174.462521:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763174.462523:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:0.0:1787763174.462527:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763174.462528:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763174.462531:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763174.462533:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763174.462545:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763174.462547:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763174.462548:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763174.462551:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832598528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763199 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.462556:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763174.462558:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832598528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763199 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.462563:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763174.462564:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763174.462567:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763174.462570:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:0.0:1787763174.462573:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763174.462574:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315561505
00000100:00000001:0.0:1787763174.462577:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315562740
00000100:00000001:0.0:1787763174.462579:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315561505
00000100:00000001:0.0:1787763174.462580:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315562740
00000100:00000001:0.0:1787763174.462582:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763174.462584:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763174.462585:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763174.462587:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763174/real 1787763174]  req@ffff9751f8658000 x1874599832598528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763174.462592:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763174.462592:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763174.462594:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763174.462596:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763174.462597:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763174.462598:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763174.462612:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832598528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763174.462618:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832598528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.462622:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763174.462623:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763174.462624:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763174.462625:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763174.462627:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832598528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.462631:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763174.462633:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832598528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.462637:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763174.462639:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763174.462640:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763174.462641:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763174.462643:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763174.462656:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763174.462658:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763174.462660:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763174.462662:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763174.462664:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832598528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763174.462669:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832598528:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763174.462671:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763174.462674:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832598656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.462678:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763174.462680:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763174.462681:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763174.462683:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763174.462685:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763174.462688:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832598656:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763174.462692:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763174.462695:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763174.462697:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763174.462699:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763174.462713:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763174.462716:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763174.462719:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763174.462722:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763174.462726:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763174.462729:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832598656, portal 10
00000100:00000001:0.0:1787763174.462731:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763174.462733:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:0.0:1787763174.462740:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832598656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763199 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.462747:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763174.462749:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763174.462751:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763174.462753:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832598656, offset 0
00000400:00000200:0.0:1787763174.462755:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763174.462759:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763174.462762:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763174.462765:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763174.462766:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763174.462769:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763174.462770:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763174.462773:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57912:57912:256:4294967295] 192.168.203.110@tcp LPNI seq info [57912:57912:8:4294967295]
00000400:00000200:0.0:1787763174.462776:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763174.462780:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763174.462783:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763174.462785:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763174.462788:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763174.462790:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787763174.462793:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763174.462795:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:0.0:1787763174.462798:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763174.462799:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763174.462802:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763174.462804:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763174.462806:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763174.462808:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763174.462809:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763174.462812:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832598656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763199 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.462817:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763174.462820:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832598656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763199 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.462825:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763174.462826:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763174.462829:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763174.462831:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:0.0:1787763174.462833:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763174.462836:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315561493
00000100:00000001:0.0:1787763174.462848:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315562740
00000100:00000001:0.0:1787763174.462851:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315561493
00000100:00000001:0.0:1787763174.462853:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315562740
00000100:00000001:0.0:1787763174.462856:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763174.462858:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763174.462860:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763174.462864:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763174/real 1787763174]  req@ffff9751f8659a40 x1874599832598656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763174.462873:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763174.462875:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763174.462877:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763174.462881:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763174.462883:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763174.462885:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763174.462888:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832598656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763174.462897:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832598656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.462902:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763174.462903:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763174.462905:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763174.462906:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763174.462911:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832598656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.462942:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763174.462947:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832598656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.462955:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763174.462958:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763174.462960:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763174.462962:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763174.462964:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763174.462966:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763174.462969:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763174.462973:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763174.462975:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763174.462978:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832598656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763174.462984:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832598656:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763174.462987:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763174.462990:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832598784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.463007:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763174.463008:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763174.463010:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763174.463011:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763174.463013:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763174.463014:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832598784:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763174.463016:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763174.463019:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763174.463020:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763174.463022:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763174.463025:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763174.463026:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763174.463029:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763174.463031:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763174.463034:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763174.463037:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832598784, portal 4
00000100:00000001:0.0:1787763174.463038:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763174.463039:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000040:0.0:1787763174.463044:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832598784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763199 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.463049:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763174.463051:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763174.463052:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763174.463054:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832598784, offset 0
00000400:00000200:0.0:1787763174.463057:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763174.463059:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763174.463062:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763174.463064:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763174.463066:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763174.463068:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763174.463070:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763174.463072:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57913:57913:256:4294967295] 192.168.203.110@tcp LPNI seq info [57913:57913:8:4294967295]
00000400:00000200:0.0:1787763174.463075:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763174.463079:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763174.463083:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763174.463085:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763174.463088:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763174.463090:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:0.1:1787763174.463093:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763174.463095:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:0.0:1787763174.463098:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763174.463101:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763174.463104:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763174.463108:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763174.463111:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763174.463113:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763174.463115:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763174.463119:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832598784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763199 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.463128:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763174.463131:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832598784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763199 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.463160:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763174.463161:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763174.463165:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763174.463167:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:0.0:1787763174.463170:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763174.463171:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315561511
00000100:00000001:0.0:1787763174.463174:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315562740
00000100:00000001:0.0:1787763174.463175:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315561511
00000100:00000001:0.0:1787763174.463177:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315562740
00000100:00000001:0.0:1787763174.463179:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763174.463180:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763174.463181:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763174.463184:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763174/real 1787763174]  req@ffff9751f8658780 x1874599832598784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763174.463189:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763174.463190:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763174.463192:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763174.463194:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763174.463195:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763174.463198:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763174.463200:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832598784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763174.463206:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832598784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.463210:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763174.463210:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763174.463212:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763174.463212:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763174.463214:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832598784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.463219:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763174.463222:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832598784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.463227:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763174.463228:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763174.463229:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763174.463230:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763174.463231:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763174.463232:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763174.463233:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763174.463236:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763174.463237:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763174.463239:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832598784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763174.463244:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832598784:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763174.463246:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763174.463247:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763174.463250:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832598528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.463255:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763174.463256:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763174.463258:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016000.
02000000:00000001:0.0:1787763174.463259:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763174.463260:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763174.463262:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763174.463263:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763174.463265:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:0.0:1787763174.463267:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763174.463268:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763174.463269:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:0.0:1787763174.463271:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763174.463272:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763174.463273:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763174.463275:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832598656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.463279:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763174.463280:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763174.463281:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763174.463282:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763174.463283:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763174.463283:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763174.463285:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763174.463286:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:0.0:1787763174.463287:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763174.463288:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763174.463288:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:0.0:1787763174.463290:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763174.463291:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763174.463291:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763174.463293:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832598784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763199 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763174.463297:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763174.463298:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763174.463300:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763174.463301:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763174.463302:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763174.463303:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787763174.463305:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763174.463306:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:0.0:1787763174.463307:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763174.463307:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763174.463308:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:0.0:1787763174.463309:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763174.463310:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763174.463311:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763174.463314:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763174.463314:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763174.463316:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763174.463317:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763177.148363:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763177.148370:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763177.148378:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763178.172318:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763178.172325:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763178.172336:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763179.580259:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763179.580271:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763179.580274:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763179.580278:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763179.580284:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763179.580294:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 20894
00000400:00000200:0.0:1787763179.580306:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763179.580316:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763179.580320:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763179.580325:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763179.580328:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763179.580331:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763179.580334:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763179.580339:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763179.580341:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763179.580344:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763179.580347:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763179.580349:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763179.580351:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763179.580354:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763179.580355:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763179.580357:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763179.580358:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763179.580360:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763179.580363:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763179.580365:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763179.580367:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763179.580370:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763179.580372:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763179.580375:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763179.580377:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763179.580387:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
00000020:00000040:0.0:1787763179.580391:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763179.580395:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763179.580396:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763179.580399:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763179.580402:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763179.580406:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763179.580414:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787763179.580421:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763179.580430:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832598912/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763179.580440:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763179.580444:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763179.580449:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832598912/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763179.580469:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763179.580472:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763179.580475:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763179.580476:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763179.580478:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763179.580481:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763179.580483:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20894
00000400:00000200:0.0:1787763179.580486:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763179.580488:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763179.580490:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763179.580493:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763179.580495:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763179.580497:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763179.580499:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763179.580501:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763179.580503:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763179.580507:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763179.580509:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763179.580511:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763179.580516:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763179.580526:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763179.580532:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763179.580540:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763179.580548:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763179.580556:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763179.580565:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763179.580574:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763179.580578:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763179.580586:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763179.580591:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763179.580593:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763179.580598:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763179.580605:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763179.580608:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763179.580616:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787763179.580621:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763179.580625:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763179.580626:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763179.580627:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763179.580629:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763179.580632:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763179.580638:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787763179.580642:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763179.580652:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832599040/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763179.580673:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763179.580678:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763179.580685:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832599040/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763179.580701:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763179.580704:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763179.580710:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763179.580713:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763179.580715:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763179.580717:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763179.580720:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20894
00000400:00000200:0.0:1787763179.580727:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763179.580734:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763179.580739:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763179.580743:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763179.580746:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763179.580753:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763179.580756:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763179.580760:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763179.580763:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763179.580766:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763179.580769:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763179.580772:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763179.580778:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763179.580786:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763179.580788:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763179.580792:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763179.580796:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763179.580801:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763179.580808:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763179.580815:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763179.580820:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763179.580823:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763179.580828:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763179.580830:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763179.580831:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763179.580837:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763179.580842:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763179.580847:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000020:00000040:0.0:1787763179.580852:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000800:00000200:3.1:1787763179.580852:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:0.0:1787763179.580857:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:3.1:1787763179.580859:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
02000000:00000001:0.0:1787763179.580860:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763179.580864:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763179.580867:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763179.580878:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763179.580900:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787763179.580906:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763179.580915:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832599168/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763179.580926:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763179.580930:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763179.580939:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832599168/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763179.580951:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763179.580957:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763179.580966:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763179.580971:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763179.580976:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763179.580978:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763179.580980:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763179.580982:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763179.580989:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20894
00000400:00000200:0.0:1787763179.580995:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763179.580999:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763179.581002:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763179.581008:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763179.581010:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763179.581012:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763179.581013:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763179.581017:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763179.581019:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763179.581020:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763179.581022:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763179.581025:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763179.581030:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763179.581035:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763179.581038:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763179.581042:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763179.581045:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763179.581049:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763179.581054:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763179.581059:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763179.581062:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763179.581067:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763179.581072:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763179.581075:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763179.581080:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763179.581084:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763179.581087:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763179.581094:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763179.581097:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763179.581103:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763179.581105:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763179.581110:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763179.581112:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763179.581116:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763179.581122:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763179.581127:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763179.581158:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832599296/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763179.581170:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763179.581175:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763179.581180:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832599296/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763179.581190:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763179.581197:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20904)
00000800:00000200:3.1:1787763179.581390:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763179.581392:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787763179.581648:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763179.581650:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787763179.581848:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763179.581850:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000020:00000001:0.0:1787763180.220312:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763180.220316:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763180.220320:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763181.244325:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763181.244333:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763181.244340:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763183.292298:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763183.292311:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763183.292322:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763184.316344:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763184.316373:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763184.316380:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763184.700657:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763184.700669:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763184.700676:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763184.700696:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bcf00 x1874599832599424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.700720:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763184.700725:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763184.700739:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763184.700747:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763184.700758:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763184.700781:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832599424:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763184.700801:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763184.700811:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763184.700816:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763184.700829:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763184.700843:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9017800.
02000000:00000001:0.0:1787763184.700861:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763184.700871:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763184.700882:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763184.700899:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763184.700919:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832599424, portal 25
00000100:00000001:0.0:1787763184.700926:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763184.700932:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049849600 : -115096659702016 : ffff9751f87bcf00)
00000100:00000040:0.0:1787763184.700946:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bcf00 x1874599832599424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763209 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.700976:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763184.700993:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763184.701002:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763184.701013:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832599424, offset 0
00000400:00000200:0.0:1787763184.701038:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763184.701053:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763184.701073:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763184.701082:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763184.701093:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763184.701108:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763184.701117:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763184.701157:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57918:57918:256:4294967295] 192.168.203.110@tcp LPNI seq info [57918:57918:8:4294967295]
00000400:00000200:0.0:1787763184.701164:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763184.701187:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763184.701195:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763184.701210:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763184.701233:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763184.701242:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787763184.701267:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763184.701279:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1787763184.701299:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763184.701311:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763184.701328:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763184.701340:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763184.701360:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763184.701371:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763184.701386:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763184.701401:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bcf00 x1874599832599424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763209 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.701430:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763184.701441:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bcf00 x1874599832599424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763209 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.701465:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763184.701482:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763184.701504:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763184.701517:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763184.701526:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315572980
00000100:00000010:0.0:1787763184.701543:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940ccc0.
00000100:00000001:0.0:1787763184.701569:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763184.701582:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315572980
00000100:00000010:0.0:1787763184.701588:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c9c0.
00000100:00000001:0.0:1787763184.701600:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787763184.701609:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763184.701623:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763184.701628:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763184.701642:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763184/real 1787763184]  req@ffff9751f87bcf00 x1874599832599424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763184.701659:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763184.701662:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763184.701668:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763184.701679:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763184.701685:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763184.701696:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763184.701702:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bcf00 x1874599832599424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763184.701722:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bcf00 x1874599832599424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.701755:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763184.701760:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.701774:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763184.701779:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763184.701794:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bcf00 x1874599832599424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.701817:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763184.701830:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bcf00 x1874599832599424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.701865:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763184.701869:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.701878:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763184.701888:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.701896:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763184.701904:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763184.701921:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763184.701938:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763184.701947:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763184.701962:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bcf00 x1874599832599424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763184.702009:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832599424:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763184.702032:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.702044:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763184.702055:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bcf00 x1874599832599424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.702077:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763184.702089:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763184.702098:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9017800.
02000000:00000001:0.0:1787763184.702120:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763184.702126:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763184.702130:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787763184.702174:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763184.702184:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612800.
02000000:00000001:0.0:1787763184.702193:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763184.702206:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763184.702219:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bcf00.
00000100:00000001:0.0:1787763184.702233:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763184.702237:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.702246:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.702257:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763184.702264:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763184.702279:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763184.702287:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763184.702292:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763184.702306:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc780 x1874599832599552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.702330:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763184.702333:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763184.702337:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763184.702340:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763184.702346:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763184.702362:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832599552:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763184.702380:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763184.702385:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763184.702390:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763184.702395:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763184.702401:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9017800.
02000000:00000001:0.0:1787763184.702411:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763184.702419:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763184.702422:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763184.702426:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763184.702430:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832599552, portal 10
00000100:00000001:0.0:1787763184.702432:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763184.702433:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049847680 : -115096659703936 : ffff9751f87bc780)
00000100:00000040:0.0:1787763184.702445:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc780 x1874599832599552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763209 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.702450:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763184.702454:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763184.702456:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763184.702458:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832599552, offset 0
00000400:00000200:0.0:1787763184.702461:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763184.702465:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763184.702468:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763184.702471:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763184.702473:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763184.702475:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763184.702477:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763184.702479:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57919:57919:256:4294967295] 192.168.203.110@tcp LPNI seq info [57919:57919:8:4294967295]
00000400:00000200:0.0:1787763184.702482:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763184.702487:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763184.702490:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763184.702493:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763184.702496:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763184.702498:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787763184.702502:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763184.702504:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1787763184.702507:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763184.702508:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763184.702511:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763184.702513:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763184.702516:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763184.702517:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763184.702519:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763184.702522:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc780 x1874599832599552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763209 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.702528:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763184.702530:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc780 x1874599832599552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763209 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.702535:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763184.702536:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763184.702539:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763184.702543:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:0.0:1787763184.702546:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763184.702548:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315571745
00000100:00000001:0.0:1787763184.702550:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315572980
00000100:00000001:0.0:1787763184.702552:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315571745
00000100:00000001:0.0:1787763184.702554:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315572980
00000100:00000001:0.0:1787763184.702555:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763184.702557:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763184.702559:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763184.702561:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763184/real 1787763184]  req@ffff9751f87bc780 x1874599832599552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763184.702566:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763184.702566:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763184.702567:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763184.702569:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763184.702570:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763184.702571:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763184.702573:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc780 x1874599832599552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763184.702579:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc780 x1874599832599552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.702584:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763184.702584:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.702586:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763184.702586:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763184.702588:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc780 x1874599832599552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.702592:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763184.702595:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc780 x1874599832599552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.702601:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763184.702603:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.702604:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763184.702605:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.702606:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763184.702620:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763184.702621:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763184.702624:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763184.702625:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763184.702628:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc780 x1874599832599552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763184.702633:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832599552:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763184.702636:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763184.702639:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc3c0 x1874599832599680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.702643:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763184.702644:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763184.702646:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763184.702647:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763184.702648:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763184.702650:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832599680:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763184.702652:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763184.702653:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763184.702654:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763184.702655:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763184.702658:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9017000.
02000000:00000001:0.0:1787763184.702660:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763184.702662:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763184.702664:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763184.702666:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763184.702668:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832599680, portal 10
00000100:00000001:0.0:1787763184.702670:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763184.702671:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049846720 : -115096659704896 : ffff9751f87bc3c0)
00000100:00000040:0.0:1787763184.702675:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc3c0 x1874599832599680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763209 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.702680:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763184.702681:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763184.702683:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763184.702684:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832599680, offset 0
00000400:00000200:0.0:1787763184.702687:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763184.702689:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763184.702691:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763184.702693:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763184.702694:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763184.702696:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763184.702698:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763184.702700:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57920:57920:256:4294967295] 192.168.203.110@tcp LPNI seq info [57920:57920:8:4294967295]
00000400:00000200:0.0:1787763184.702703:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763184.702707:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763184.702710:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763184.702712:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763184.702714:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763184.702715:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787763184.702718:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763184.702720:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1787763184.702723:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763184.702724:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763184.702726:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763184.702729:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763184.702730:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763184.702733:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763184.702734:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763184.702736:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc3c0 x1874599832599680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763209 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.702742:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763184.702744:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc3c0 x1874599832599680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763209 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.702749:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763184.702750:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763184.702752:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763184.702753:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:0.0:1787763184.702755:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763184.702756:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315571733
00000100:00000001:0.0:1787763184.702758:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315572980
00000100:00000001:0.0:1787763184.702759:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315571733
00000100:00000001:0.0:1787763184.702761:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315572980
00000100:00000001:0.0:1787763184.702763:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763184.702765:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763184.702766:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763184.702768:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763184/real 1787763184]  req@ffff9751f87bc3c0 x1874599832599680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763184.702772:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763184.702773:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763184.702774:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763184.702776:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763184.702776:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763184.702778:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763184.702779:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc3c0 x1874599832599680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763184.702785:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc3c0 x1874599832599680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.702790:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763184.702791:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.702793:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763184.702794:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763184.702796:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc3c0 x1874599832599680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.702800:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763184.702802:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc3c0 x1874599832599680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.702806:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763184.702807:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.702808:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763184.702809:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.702810:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763184.702811:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763184.702813:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763184.702814:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763184.702816:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763184.702818:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc3c0 x1874599832599680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763184.702822:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832599680:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763184.702825:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.702827:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763184.702828:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc780 x1874599832599552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.702833:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763184.702835:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763184.702836:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9017800.
02000000:00000001:0.0:1787763184.702838:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763184.702840:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763184.702841:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763184.702843:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763184.702860:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612400.
02000000:00000001:0.0:1787763184.702864:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763184.702865:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763184.702867:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc780.
00000100:00000001:0.0:1787763184.702870:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763184.702871:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.702873:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763184.702877:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc3c0 x1874599832599680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.702884:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763184.702886:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763184.702888:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9017000.
02000000:00000001:0.0:1787763184.702890:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763184.702892:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763184.702893:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763184.702895:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763184.702896:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613000.
02000000:00000001:0.0:1787763184.702899:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763184.702900:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763184.702902:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc3c0.
00000100:00000001:0.0:1787763184.702904:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763184.702906:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.702908:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.702911:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763184.702912:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763184.702914:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763184.702915:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763184.702916:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763184.702920:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bfc00 x1874599832599808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.702925:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763184.702927:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763184.702929:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763184.702930:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763184.702932:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763184.702933:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832599808:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763184.702935:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763184.702937:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763184.702938:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763184.702939:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763184.702942:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9017000.
02000000:00000001:0.0:1787763184.702943:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763184.702947:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763184.702948:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763184.702951:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763184.702954:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832599808, portal 4
00000100:00000001:0.0:1787763184.702955:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763184.702956:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049861120 : -115096659690496 : ffff9751f87bfc00)
00000100:00000040:0.0:1787763184.702961:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bfc00 x1874599832599808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763209 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.702966:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763184.702967:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763184.702969:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763184.702970:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832599808, offset 0
00000400:00000200:0.0:1787763184.702973:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763184.702975:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763184.702977:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763184.702980:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763184.702982:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763184.702984:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763184.702986:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763184.702989:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57921:57921:256:4294967295] 192.168.203.110@tcp LPNI seq info [57921:57921:8:4294967295]
00000400:00000200:0.0:1787763184.702994:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763184.703000:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763184.703006:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763184.703009:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763184.703014:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763184.703017:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:0.1:1787763184.703022:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763184.703026:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:0.0:1787763184.703030:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763184.703032:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763184.703035:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763184.703039:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763184.703041:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763184.703043:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763184.703045:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763184.703048:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bfc00 x1874599832599808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763209 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.703053:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763184.703056:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bfc00 x1874599832599808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763209 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.703060:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763184.703062:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763184.703064:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763184.703067:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:0.0:1787763184.703070:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763184.703072:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315571750
00000100:00000001:0.0:1787763184.703074:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315572980
00000100:00000001:0.0:1787763184.703076:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315571750
00000100:00000001:0.0:1787763184.703077:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315572980
00000100:00000001:0.0:1787763184.703079:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763184.703080:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763184.703081:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763184.703083:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763184/real 1787763184]  req@ffff9751f87bfc00 x1874599832599808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763184.703089:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763184.703090:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763184.703092:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763184.703094:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763184.703095:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763184.703096:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763184.703098:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bfc00 x1874599832599808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763184.703104:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bfc00 x1874599832599808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.703109:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763184.703110:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.703111:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763184.703112:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763184.703114:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bfc00 x1874599832599808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.703118:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763184.703120:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bfc00 x1874599832599808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.703124:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763184.703125:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.703126:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763184.703127:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.703128:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763184.703129:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763184.703131:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763184.703150:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763184.703152:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763184.703156:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bfc00 x1874599832599808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763184.703161:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832599808:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763184.703164:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.703165:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763184.703167:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bfc00 x1874599832599808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763209 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763184.703171:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763184.703172:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763184.703174:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9017000.
02000000:00000001:0.0:1787763184.703176:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763184.703177:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763184.703178:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787763184.703179:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763184.703181:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612c00.
02000000:00000001:0.0:1787763184.703183:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763184.703184:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763184.703185:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bfc00.
00000100:00000001:0.0:1787763184.703187:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763184.703187:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.703189:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763184.703191:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763184.703192:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763184.703193:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763184.703195:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763186.364385:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763186.364392:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763186.364398:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763187.388401:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763187.388408:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763187.388413:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763188.412462:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763188.412471:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763188.412477:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763189.436379:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763189.436385:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763189.436390:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763189.820768:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763189.820778:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763189.820785:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763189.820801:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832599936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.820843:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763189.820849:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763189.820857:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763189.820863:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763189.820872:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763189.820881:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832599936:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763189.820892:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763189.820903:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763189.820912:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763189.820921:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763189.820935:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9017000.
02000000:00000001:0.0:1787763189.820946:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763189.820965:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763189.820976:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763189.820994:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763189.821007:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832599936, portal 25
00000100:00000001:0.0:1787763189.821017:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763189.821022:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:0.0:1787763189.821038:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832599936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763214 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.821065:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763189.821077:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763189.821083:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763189.821092:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832599936, offset 0
00000400:00000200:0.0:1787763189.821109:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763189.821128:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763189.821175:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763189.821183:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763189.821192:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763189.821204:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763189.821217:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763189.821237:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57922:57922:256:4294967295] 192.168.203.110@tcp LPNI seq info [57922:57922:8:4294967295]
00000400:00000200:0.0:1787763189.821264:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763189.821297:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763189.821319:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763189.821329:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763189.821348:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763189.821357:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787763189.821371:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763189.821380:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1787763189.821394:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763189.821401:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763189.821411:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763189.821425:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763189.821435:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763189.821441:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763189.821452:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763189.821466:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832599936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763214 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.821503:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763189.821529:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832599936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763214 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.821556:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763189.821562:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763189.821577:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763189.821590:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763189.821598:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315578100
00000100:00000010:0.0:1787763189.821610:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c7e0.
00000100:00000001:0.0:1787763189.821626:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763189.821642:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315578100
00000100:00000010:0.0:1787763189.821653:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c4e0.
00000100:00000001:0.0:1787763189.821662:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787763189.821674:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763189.821687:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763189.821700:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763189.821721:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763189/real 1787763189]  req@ffff9751f86583c0 x1874599832599936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763189.821773:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763189.821779:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763189.821787:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763189.821801:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763189.821837:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763189.821846:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763189.821860:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832599936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763189.821898:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832599936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.821932:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763189.821939:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763189.821951:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763189.821960:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763189.821978:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832599936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.822003:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763189.822016:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832599936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.822042:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763189.822052:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763189.822059:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763189.822064:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763189.822071:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763189.822077:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763189.822083:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763189.822095:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763189.822105:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763189.822115:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832599936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763189.822174:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832599936:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763189.822189:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763189.822197:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763189.822213:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832599936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.822251:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763189.822259:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763189.822265:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9017000.
02000000:00000001:0.0:1787763189.822275:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763189.822280:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763189.822284:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787763189.822289:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763189.822294:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:0.0:1787763189.822301:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763189.822303:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763189.822306:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:0.0:1787763189.822310:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763189.822312:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763189.822314:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763189.822323:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763189.822326:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763189.822329:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763189.822331:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763189.822333:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763189.822341:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832600064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.822350:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763189.822353:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763189.822356:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763189.822359:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763189.822361:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763189.822365:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832600064:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763189.822370:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763189.822373:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763189.822375:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763189.822378:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763189.822382:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9017000.
02000000:00000001:0.0:1787763189.822385:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763189.822390:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763189.822394:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763189.822399:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763189.822404:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832600064, portal 10
00000100:00000001:0.0:1787763189.822407:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763189.822408:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000040:0.0:1787763189.822417:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832600064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763214 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.822426:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763189.822429:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763189.822432:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763189.822435:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832600064, offset 0
00000400:00000200:0.0:1787763189.822440:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763189.822446:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763189.822451:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763189.822456:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763189.822460:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763189.822463:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763189.822467:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763189.822472:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57923:57923:256:4294967295] 192.168.203.110@tcp LPNI seq info [57923:57923:8:4294967295]
00000400:00000200:0.0:1787763189.822477:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763189.822487:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763189.822492:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763189.822497:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763189.822502:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763189.822507:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787763189.822512:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763189.822516:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1787763189.822523:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763189.822527:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763189.822533:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763189.822541:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763189.822547:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763189.822551:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763189.822556:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763189.822564:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832600064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763214 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.822581:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763189.822587:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832600064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763214 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.822602:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763189.822606:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763189.822613:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763189.822621:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:0.0:1787763189.822628:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763189.822632:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315576865
00000100:00000001:0.0:1787763189.822638:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315578100
00000100:00000001:0.0:1787763189.822643:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315576865
00000100:00000001:0.0:1787763189.822648:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315578100
00000100:00000001:0.0:1787763189.822653:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763189.822657:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763189.822661:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763189.822669:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763189/real 1787763189]  req@ffff9751f8658780 x1874599832600064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763189.822687:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763189.822690:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763189.822693:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763189.822700:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763189.822704:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763189.822708:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763189.822714:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832600064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763189.822726:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832600064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.822746:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763189.822748:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763189.822750:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763189.822752:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763189.822756:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832600064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.822765:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763189.822770:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832600064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.822779:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763189.822783:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763189.822784:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763189.822786:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763189.822788:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763189.822835:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763189.822840:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763189.822845:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763189.822848:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763189.822853:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832600064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763189.822864:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832600064:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763189.822869:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763189.822873:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832600192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.822881:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763189.822883:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763189.822885:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763189.822888:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763189.822891:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763189.822894:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832600192:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763189.822901:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763189.822905:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763189.822908:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763189.822911:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763189.822918:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9017800.
02000000:00000001:0.0:1787763189.822922:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763189.822928:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763189.822933:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763189.822940:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763189.822946:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832600192, portal 10
00000100:00000001:0.0:1787763189.822951:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763189.822954:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:0.0:1787763189.822966:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832600192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763214 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.822993:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763189.822998:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763189.823003:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763189.823010:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832600192, offset 0
00000400:00000200:0.0:1787763189.823015:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763189.823021:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763189.823027:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763189.823031:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763189.823035:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763189.823038:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763189.823041:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763189.823046:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57924:57924:256:4294967295] 192.168.203.110@tcp LPNI seq info [57924:57924:8:4294967295]
00000400:00000200:0.0:1787763189.823051:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763189.823060:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763189.823066:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763189.823071:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763189.823076:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763189.823080:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787763189.823085:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763189.823088:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1787763189.823093:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763189.823096:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763189.823100:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763189.823104:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763189.823108:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763189.823111:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763189.823114:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763189.823119:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832600192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763214 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.823129:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763189.823162:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832600192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763214 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.823181:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763189.823183:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763189.823189:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763189.823193:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:0.0:1787763189.823198:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763189.823201:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315576854
00000100:00000001:0.0:1787763189.823205:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315578100
00000100:00000001:0.0:1787763189.823209:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315576854
00000100:00000001:0.0:1787763189.823214:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315578100
00000100:00000001:0.0:1787763189.823219:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763189.823223:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763189.823226:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763189.823234:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763189/real 1787763189]  req@ffff9751f8659a40 x1874599832600192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763189.823252:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763189.823255:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763189.823258:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763189.823264:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763189.823268:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763189.823272:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763189.823280:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832600192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763189.823299:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832600192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.823312:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763189.823314:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763189.823317:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763189.823319:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763189.823324:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832600192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.823333:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763189.823338:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832600192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.823347:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763189.823350:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763189.823352:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763189.823353:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763189.823356:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763189.823358:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763189.823361:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763189.823365:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763189.823367:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763189.823372:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832600192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763189.823381:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832600192:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763189.823386:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763189.823391:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832600320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.823399:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763189.823401:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763189.823404:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763189.823406:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763189.823409:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763189.823412:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832600320:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763189.823416:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763189.823419:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763189.823422:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763189.823424:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763189.823429:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016800.
02000000:00000001:0.0:1787763189.823432:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763189.823437:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763189.823440:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763189.823445:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763189.823449:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832600320, portal 4
00000100:00000001:0.0:1787763189.823451:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763189.823453:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:0.0:1787763189.823461:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832600320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763214 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.823469:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763189.823472:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763189.823474:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763189.823477:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832600320, offset 0
00000400:00000200:0.0:1787763189.823482:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763189.823487:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763189.823492:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763189.823495:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763189.823499:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763189.823502:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763189.823505:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763189.823509:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57925:57925:256:4294967295] 192.168.203.110@tcp LPNI seq info [57925:57925:8:4294967295]
00000400:00000200:0.0:1787763189.823514:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763189.823523:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763189.823528:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763189.823532:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763189.823537:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763189.823540:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:0.1:1787763189.823545:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763189.823548:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:0.0:1787763189.823553:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763189.823557:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763189.823560:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763189.823565:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763189.823569:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763189.823571:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763189.823574:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763189.823578:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832600320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763214 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.823588:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763189.823592:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832600320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763214 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.823600:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763189.823602:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763189.823606:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763189.823610:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:0.0:1787763189.823614:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763189.823616:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315576871
00000100:00000001:0.0:1787763189.823619:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315578100
00000100:00000001:0.0:1787763189.823622:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315576871
00000100:00000001:0.0:1787763189.823625:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315578100
00000100:00000001:0.0:1787763189.823627:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763189.823630:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763189.823632:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763189.823636:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763189/real 1787763189]  req@ffff9751f8658000 x1874599832600320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763189.823645:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763189.823647:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763189.823648:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763189.823652:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763189.823653:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763189.823656:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763189.823660:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832600320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763189.823671:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832600320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.823678:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763189.823680:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763189.823682:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763189.823684:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763189.823688:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832600320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.823696:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763189.823700:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832600320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.823708:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763189.823711:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763189.823713:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763189.823714:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763189.823716:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763189.823718:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763189.823721:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763189.823725:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763189.823728:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763189.823733:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832600320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763189.823743:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832600320:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763189.823747:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763189.823749:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763189.823754:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832600064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.823763:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763189.823765:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763189.823767:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9017000.
02000000:00000001:0.0:1787763189.823771:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763189.823773:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763189.823775:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763189.823777:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763189.823780:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:0.0:1787763189.823783:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763189.823785:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763189.823787:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:0.0:1787763189.823790:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763189.823792:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763189.823794:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763189.823798:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832600192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.823805:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763189.823807:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763189.823836:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9017800.
02000000:00000001:0.0:1787763189.823842:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763189.823844:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763189.823846:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763189.823848:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763189.823850:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:0.0:1787763189.823853:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763189.823854:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763189.823856:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:0.0:1787763189.823859:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763189.823860:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763189.823863:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763189.823867:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832600320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763214 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763189.823875:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763189.823877:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763189.823879:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016800.
02000000:00000001:0.0:1787763189.823883:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763189.823885:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763189.823886:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787763189.823889:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763189.823891:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:0.0:1787763189.823893:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763189.823894:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763189.823896:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:0.0:1787763189.823899:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763189.823901:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763189.823905:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763189.823912:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763189.823914:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763189.823918:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763189.823921:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763190.460489:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763190.460494:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763190.460498:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763191.484439:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763191.484446:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763191.484451:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763192.508279:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763192.508283:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763192.508286:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763193.532348:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763193.532353:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763193.532356:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763194.556309:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763194.556315:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763194.556320:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763194.940657:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763194.940666:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763194.940671:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763194.940692:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832600448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.940798:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763194.940807:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763194.940819:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763194.940826:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763194.940834:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763194.940848:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832600448:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763194.940861:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763194.940870:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763194.940875:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763194.940880:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763194.940893:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016800.
02000000:00000001:0.0:1787763194.940900:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763194.940914:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763194.940925:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763194.940941:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763194.940955:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832600448, portal 25
00000100:00000001:0.0:1787763194.940968:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763194.940978:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:0.0:1787763194.941005:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832600448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763219 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.941040:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763194.941057:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763194.941067:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763194.941081:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832600448, offset 0
00000400:00000200:0.0:1787763194.941098:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763194.941119:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763194.941171:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763194.941179:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763194.941188:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763194.941198:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763194.941210:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763194.941224:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57926:57926:256:4294967295] 192.168.203.110@tcp LPNI seq info [57926:57926:8:4294967295]
00000400:00000200:0.0:1787763194.941246:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763194.941291:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763194.941321:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763194.941338:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763194.941359:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763194.941369:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787763194.941386:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763194.941397:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1787763194.941419:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763194.941428:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763194.941441:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763194.941458:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763194.941470:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763194.941477:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763194.941485:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763194.941498:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832600448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763219 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.941530:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763194.941543:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832600448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763219 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.941575:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763194.941585:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763194.941601:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763194.941616:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763194.941624:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315583220
00000100:00000010:0.0:1787763194.941638:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940ca80.
00000100:00000001:0.0:1787763194.941652:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763194.941667:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315583220
00000100:00000010:0.0:1787763194.941677:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c360.
00000100:00000001:0.0:1787763194.941689:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787763194.941704:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763194.941722:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763194.941737:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763194.941758:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763194/real 1787763194]  req@ffff9751f865ad00 x1874599832600448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763194.941826:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763194.941832:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763194.941839:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763194.941849:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763194.941858:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763194.941868:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763194.941883:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832600448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763194.941933:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832600448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.941951:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763194.941956:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763194.941962:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763194.941966:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763194.941976:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832600448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.941991:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763194.942002:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832600448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.942018:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763194.942026:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763194.942031:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763194.942033:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763194.942039:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763194.942044:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763194.942051:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763194.942059:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763194.942063:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763194.942071:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832600448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763194.942085:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832600448:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763194.942091:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763194.942095:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763194.942099:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832600448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.942109:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763194.942112:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763194.942115:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016800.
02000000:00000001:0.0:1787763194.942121:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763194.942123:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763194.942125:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787763194.942128:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763194.942131:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:0.0:1787763194.942176:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763194.942179:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763194.942182:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:0.0:1787763194.942188:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763194.942190:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763194.942193:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763194.942202:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763194.942205:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763194.942208:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763194.942211:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763194.942214:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763194.942223:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832600576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.942235:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763194.942238:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763194.942244:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763194.942249:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763194.942255:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763194.942263:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832600576:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763194.942271:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763194.942277:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763194.942281:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763194.942286:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763194.942294:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016800.
02000000:00000001:0.0:1787763194.942299:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763194.942310:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763194.942317:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763194.942325:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763194.942333:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832600576, portal 10
00000100:00000001:0.0:1787763194.942337:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763194.942339:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:0.0:1787763194.942349:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832600576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763219 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.942357:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763194.942361:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763194.942364:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763194.942367:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832600576, offset 0
00000400:00000200:0.0:1787763194.942373:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763194.942379:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763194.942385:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763194.942389:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763194.942408:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763194.942411:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763194.942415:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763194.942421:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57927:57927:256:4294967295] 192.168.203.110@tcp LPNI seq info [57927:57927:8:4294967295]
00000400:00000200:0.0:1787763194.942426:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763194.942437:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763194.942444:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763194.942451:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763194.942462:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763194.942466:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787763194.942472:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763194.942476:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1787763194.942482:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763194.942485:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763194.942491:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763194.942495:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763194.942499:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763194.942502:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763194.942505:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763194.942511:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832600576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763219 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.942522:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763194.942526:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832600576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763219 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.942536:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763194.942538:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763194.942545:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763194.942551:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:0.0:1787763194.942555:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763194.942557:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315581985
00000100:00000001:0.0:1787763194.942561:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315583220
00000100:00000001:0.0:1787763194.942564:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315581985
00000100:00000001:0.0:1787763194.942567:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315583220
00000100:00000001:0.0:1787763194.942571:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763194.942574:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763194.942577:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763194.942583:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763194/real 1787763194]  req@ffff9751f8658000 x1874599832600576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763194.942612:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763194.942615:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763194.942618:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763194.942625:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763194.942628:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763194.942633:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763194.942642:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832600576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763194.942664:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832600576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.942680:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763194.942684:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763194.942688:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763194.942691:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763194.942702:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832600576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.942718:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763194.942728:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832600576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.942744:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763194.942752:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763194.942757:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763194.942762:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763194.942767:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763194.942812:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763194.942822:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763194.942830:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763194.942836:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763194.942846:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832600576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763194.942859:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832600576:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763194.942867:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763194.942873:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832600704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.942882:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763194.942885:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763194.942890:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763194.942893:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763194.942897:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763194.942903:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832600704:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763194.942909:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763194.942913:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763194.942916:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763194.942919:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763194.942924:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9017800.
02000000:00000001:0.0:1787763194.942927:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763194.942934:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763194.942939:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763194.942944:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763194.942949:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832600704, portal 10
00000100:00000001:0.0:1787763194.942952:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763194.942953:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:0.0:1787763194.942965:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832600704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763219 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.942982:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763194.943006:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763194.943013:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763194.943027:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832600704, offset 0
00000400:00000200:0.0:1787763194.943039:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763194.943048:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763194.943056:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763194.943062:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763194.943068:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763194.943073:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763194.943077:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763194.943081:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57928:57928:256:4294967295] 192.168.203.110@tcp LPNI seq info [57928:57928:8:4294967295]
00000400:00000200:0.0:1787763194.943087:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763194.943096:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763194.943102:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763194.943107:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763194.943113:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763194.943116:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787763194.943122:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763194.943125:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1787763194.943130:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763194.943133:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763194.943169:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763194.943177:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763194.943183:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763194.943186:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763194.943192:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763194.943199:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832600704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763219 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.943212:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763194.943219:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832600704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763219 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.943228:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763194.943230:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763194.943237:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763194.943243:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:0.0:1787763194.943250:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763194.943254:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315581974
00000100:00000001:0.0:1787763194.943259:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315583220
00000100:00000001:0.0:1787763194.943263:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315581974
00000100:00000001:0.0:1787763194.943267:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315583220
00000100:00000001:0.0:1787763194.943271:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763194.943276:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763194.943279:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763194.943288:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763194/real 1787763194]  req@ffff9751f8659a40 x1874599832600704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763194.943306:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763194.943309:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763194.943313:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763194.943319:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763194.943323:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763194.943327:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763194.943335:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832600704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763194.943348:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832600704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.943356:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763194.943358:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763194.943361:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763194.943363:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763194.943367:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832600704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.943376:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763194.943381:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832600704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.943390:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763194.943394:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763194.943396:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763194.943398:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763194.943401:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763194.943403:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763194.943406:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763194.943410:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763194.943413:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763194.943417:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832600704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763194.943427:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832600704:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763194.943431:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763194.943435:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832600832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.943444:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763194.943446:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763194.943449:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763194.943451:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763194.943454:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763194.943457:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832600832:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763194.943461:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763194.943464:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763194.943466:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763194.943469:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763194.943474:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9017000.
02000000:00000001:0.0:1787763194.943476:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763194.943481:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763194.943484:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763194.943489:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763194.943493:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832600832, portal 4
00000100:00000001:0.0:1787763194.943496:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763194.943498:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000040:0.0:1787763194.943505:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832600832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763219 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.943514:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763194.943517:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763194.943520:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763194.943523:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832600832, offset 0
00000400:00000200:0.0:1787763194.943528:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763194.943533:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763194.943538:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763194.943542:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763194.943545:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763194.943549:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763194.943551:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763194.943556:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57929:57929:256:4294967295] 192.168.203.110@tcp LPNI seq info [57929:57929:8:4294967295]
00000400:00000200:0.0:1787763194.943561:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763194.943569:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763194.943574:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763194.943578:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763194.943583:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763194.943587:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:0.1:1787763194.943592:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763194.943595:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:0.0:1787763194.943601:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763194.943603:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763194.943607:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763194.943612:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763194.943615:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763194.943618:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763194.943620:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763194.943625:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832600832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763219 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.943643:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763194.943647:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832600832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763219 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.943655:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763194.943657:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763194.943661:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763194.943665:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:0.0:1787763194.943669:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763194.943671:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315581991
00000100:00000001:0.0:1787763194.943674:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315583220
00000100:00000001:0.0:1787763194.943677:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315581991
00000100:00000001:0.0:1787763194.943680:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315583220
00000100:00000001:0.0:1787763194.943682:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763194.943685:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763194.943687:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763194.943691:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763194/real 1787763194]  req@ffff9751f8658780 x1874599832600832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763194.943700:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763194.943702:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763194.943704:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763194.943708:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763194.943710:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763194.943712:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763194.943716:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832600832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763194.943727:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832600832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.943735:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763194.943737:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763194.943740:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763194.943742:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763194.943746:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832600832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.943754:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763194.943759:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832600832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.943766:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763194.943769:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763194.943771:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763194.943773:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763194.943802:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763194.943805:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763194.943808:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763194.943813:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763194.943815:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763194.943821:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832600832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763194.943830:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832600832:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763194.943836:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763194.943838:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763194.943842:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832600576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.943851:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763194.943853:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763194.943856:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016800.
02000000:00000001:0.0:1787763194.943860:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763194.943862:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763194.943865:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763194.943867:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763194.943870:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:0.0:1787763194.943873:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763194.943875:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763194.943877:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:0.0:1787763194.943880:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763194.943882:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763194.943884:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763194.943888:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832600704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.943896:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763194.943897:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763194.943899:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9017800.
02000000:00000001:0.0:1787763194.943903:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763194.943904:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763194.943906:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763194.943908:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763194.943910:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:0.0:1787763194.943912:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763194.943913:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763194.943915:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:0.0:1787763194.943917:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763194.943919:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763194.943921:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763194.943924:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832600832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763219 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763194.943931:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763194.943933:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763194.943935:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9017000.
02000000:00000001:0.0:1787763194.943937:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763194.943938:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763194.943940:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787763194.943942:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763194.943943:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:0.0:1787763194.943945:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763194.943948:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763194.943949:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:0.0:1787763194.943952:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763194.943954:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763194.943956:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763194.943962:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763194.943965:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763194.943969:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763194.943973:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763196.604405:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763196.604410:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763196.604414:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763197.628413:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763197.628418:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763197.628435:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763200.060349:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763200.060363:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763200.060373:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763200.060383:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763200.060398:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763200.060415:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 20914
00000400:00000200:0.0:1787763200.060433:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763200.060447:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763200.060454:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763200.060462:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763200.060478:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763200.060483:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763200.060488:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763200.060495:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763200.060499:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763200.060504:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763200.060510:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763200.060515:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763200.060520:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763200.060527:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763200.060531:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763200.060535:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763200.060539:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763200.060544:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763200.060550:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763200.060554:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763200.060559:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763200.060566:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763200.060571:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763200.060575:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763200.060580:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763200.060593:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763200.060600:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763200.060606:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763200.060610:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763200.060614:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763200.060619:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763200.060625:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763200.060660:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763200.060680:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763200.060697:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832600960/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763200.060717:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763200.060724:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763200.060732:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832600960/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763200.060797:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763200.060804:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763200.060810:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763200.060813:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763200.060818:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763200.060825:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763200.060833:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20914
00000400:00000200:0.0:1787763200.060844:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763200.060852:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763200.060859:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763200.060876:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763200.060888:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763200.060897:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763200.060902:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763200.060908:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763200.060914:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763200.060921:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763200.060929:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763200.060947:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763200.060960:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763200.060977:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763200.060986:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763200.060999:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763200.061012:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763200.061028:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763200.061043:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763200.061056:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763200.061068:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763200.061081:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763200.061098:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763200.061113:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763200.061123:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763200.061177:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763200.061187:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763200.061205:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763200.061215:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763200.061221:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763200.061226:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763200.061239:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763200.061243:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763200.061247:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763200.061259:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763200.061272:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763200.061300:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832601088/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763200.061344:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763200.061354:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763200.061381:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832601088/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763200.061405:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763200.061409:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763200.061414:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763200.061416:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763200.061418:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763200.061421:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763200.061426:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20914
00000400:00000200:0.0:1787763200.061436:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763200.061443:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763200.061448:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763200.061453:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763200.061458:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763200.061462:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763200.061466:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763200.061471:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763200.061474:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763200.061477:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763200.061481:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763200.061485:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763200.061489:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763200.061495:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763200.061498:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763200.061503:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763200.061508:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763200.061513:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763200.061519:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763200.061525:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763200.061528:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763200.061534:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763200.061538:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763200.061542:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763200.061545:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763200.061550:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763200.061554:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763200.061565:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763200.061570:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763200.061575:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763200.061577:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763200.061582:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763200.061584:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763200.061589:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763200.061599:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763200.061607:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763200.061618:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832601216/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763200.061640:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763200.061645:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763200.061665:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832601216/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763200.061710:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763200.061725:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763200.061789:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763200.061808:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763200.061825:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763200.061832:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:3.1:1787763200.061846:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00080000:0.0:1787763200.061848:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:3.1:1787763200.061850:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:0.0:1787763200.061869:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763200.061888:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20914
00000400:00000200:0.0:1787763200.061908:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763200.061935:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763200.061946:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763200.061965:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763200.061973:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763200.061985:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763200.061998:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763200.062009:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763200.062015:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763200.062023:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763200.062031:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763200.062047:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763200.062064:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763200.062082:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763200.062091:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763200.062103:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763200.062121:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763200.062176:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763200.062188:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763200.062204:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763200.062215:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763200.062229:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763200.062243:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763200.062258:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763200.062269:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763200.062283:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763200.062294:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763200.062308:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763200.062320:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763200.062330:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763200.062336:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763200.062340:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763200.062345:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763200.062351:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763200.062359:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763200.062368:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763200.062385:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832601344/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763200.062415:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763200.062421:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763200.062443:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832601344/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763200.062504:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763200.062527:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20924)
00000800:00000200:3.1:1787763200.062767:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763200.062769:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787763200.063022:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763200.063024:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787763200.063246:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763200.063248:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:0.0:1787763200.700508:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763200.700515:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763200.700519:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763202.748463:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763202.748470:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763202.748476:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763203.772432:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763203.772438:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763203.772443:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763205.180277:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763205.180288:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763205.180297:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763205.180304:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763205.180311:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763205.180325:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 20919
00000400:00000200:0.0:1787763205.180344:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763205.180355:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763205.180360:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763205.180365:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763205.180370:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763205.180374:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763205.180377:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763205.180383:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763205.180385:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763205.180388:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763205.180393:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763205.180395:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763205.180397:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763205.180399:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763205.180401:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763205.180402:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763205.180404:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763205.180406:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763205.180409:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763205.180410:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763205.180414:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763205.180418:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763205.180421:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763205.180423:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763205.180426:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763205.180437:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000020:00000040:0.0:1787763205.180441:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763205.180447:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763205.180449:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763205.180453:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763205.180457:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763205.180464:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763205.180478:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787763205.180488:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763205.180503:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832601472/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763205.180517:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763205.180532:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763205.180539:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832601472/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763205.180570:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763205.180573:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763205.180577:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763205.180578:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763205.180580:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763205.180584:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763205.180587:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20919
00000400:00000200:0.0:1787763205.180592:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763205.180596:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763205.180599:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763205.180602:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763205.180605:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763205.180608:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763205.180610:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763205.180613:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763205.180615:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763205.180618:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763205.180622:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763205.180625:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763205.180628:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763205.180632:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763205.180634:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763205.180635:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763205.180637:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763205.180639:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763205.180643:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763205.180644:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763205.180646:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763205.180653:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763205.180659:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763205.180662:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763205.180670:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763205.180676:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763205.180678:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763205.180689:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000020:00000040:0.0:1787763205.180697:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763205.180721:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763205.180724:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763205.180730:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763205.180734:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763205.180742:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763205.180760:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763205.180768:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763205.180782:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832601600/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763205.180816:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763205.180827:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763205.180841:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832601600/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763205.180868:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763205.180873:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763205.180886:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763205.180890:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763205.180893:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763205.180902:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763205.180908:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20919
00000400:00000200:0.0:1787763205.180926:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763205.180933:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763205.180947:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763205.180952:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763205.180968:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763205.180980:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763205.180993:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763205.180995:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763205.181001:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763205.181006:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763205.181014:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763205.181025:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763205.181038:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763205.181049:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763205.181064:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763205.181078:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763205.181083:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763205.181095:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763205.181105:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763205.181108:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763205.181110:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763205.181119:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763205.181124:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763205.181125:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763205.181128:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763205.181158:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763205.181162:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763205.181166:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
00000020:00000040:0.0:1787763205.181168:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763205.181173:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763205.181175:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763205.181177:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763205.181179:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763205.181181:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763205.181191:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763205.181197:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763205.181206:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832601728/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763205.181222:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763205.181225:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763205.181234:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832601728/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763205.181256:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763205.181263:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763205.181272:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763205.181276:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763205.181293:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763205.181297:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763205.181302:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763205.181312:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763205.181319:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20919
00000400:00000200:0.0:1787763205.181336:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763205.181357:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763205.181372:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763205.181383:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763205.181390:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763205.181399:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763205.181404:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763205.181419:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763205.181422:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000800:00000200:3.1:1787763205.181429:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787763205.181430:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763205.181440:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000800:00000200:3.1:1787763205.181441:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:0.0:1787763205.181457:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763205.181475:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763205.181489:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763205.181501:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763205.181518:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763205.181529:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763205.181543:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763205.181561:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763205.181578:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763205.181591:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763205.181601:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763205.181611:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763205.181627:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763205.181634:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763205.181642:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763205.181648:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763205.181658:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763205.181669:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763205.181680:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763205.181686:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763205.181692:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763205.181698:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763205.181723:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763205.181731:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763205.181742:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763205.181762:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832601856/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763205.181785:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763205.181791:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763205.181802:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832601856/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763205.181859:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763205.181870:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20929)
00000800:00000200:3.1:1787763205.182325:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763205.182328:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787763205.182739:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763205.182742:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787763205.183101:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763205.183104:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:0.0:1787763205.820376:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763205.820380:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763205.820383:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763206.844423:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763206.844440:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763206.844445:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763208.892443:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763208.892452:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763208.892458:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763209.916372:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763209.916379:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763209.916385:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763210.300236:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763210.300248:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763210.300252:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763210.300258:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763210.300265:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763210.300275:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 20924
00000400:00000200:0.0:1787763210.300303:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763210.300314:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763210.300319:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763210.300331:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763210.300340:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763210.300349:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763210.300355:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763210.300364:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763210.300369:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763210.300375:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763210.300383:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763210.300390:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763210.300394:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763210.300398:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763210.300401:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763210.300405:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763210.300407:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763210.300411:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763210.300416:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763210.300419:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763210.300423:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763210.300429:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763210.300433:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763210.300436:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763210.300439:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763210.300447:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763210.300451:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763210.300456:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763210.300459:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763210.300463:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763210.300467:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763210.300471:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763210.300481:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763210.300489:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763210.300501:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832601984/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763210.300515:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763210.300521:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763210.300527:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832601984/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763210.300557:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763210.300563:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763210.300570:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763210.300573:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763210.300576:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763210.300582:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763210.300587:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20924
00000400:00000200:0.0:1787763210.300595:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763210.300603:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763210.300609:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763210.300617:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763210.300622:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763210.300628:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763210.300634:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763210.300645:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763210.300652:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763210.300662:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763210.300690:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763210.300700:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763210.300710:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763210.300724:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763210.300732:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763210.300740:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763210.300744:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763210.300752:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763210.300764:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763210.300772:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763210.300778:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763210.300791:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763210.300798:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763210.300805:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763210.300815:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763210.300819:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763210.300824:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763210.300836:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763210.300842:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763210.300858:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763210.300867:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763210.300876:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763210.300884:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763210.300893:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763210.300909:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763210.300924:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763210.300948:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832602112/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763210.300973:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763210.300982:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763210.300997:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832602112/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763210.301032:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763210.301041:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763210.301052:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763210.301058:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763210.301064:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763210.301073:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763210.301085:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20924
00000400:00000200:0.0:1787763210.301102:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763210.301114:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763210.301121:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763210.301132:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763210.301182:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763210.301191:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763210.301200:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763210.301208:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763210.301213:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763210.301220:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763210.301224:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763210.301234:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763210.301242:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:3.1:1787763210.301243:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763210.301251:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:3.1:1787763210.301254:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000100:00000001:0.0:1787763210.301257:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763210.301266:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763210.301274:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763210.301282:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763210.301292:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763210.301297:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763210.301306:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763210.301313:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763210.301320:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763210.301325:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763210.301333:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763210.301337:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763210.301346:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763210.301353:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763210.301360:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763210.301365:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763210.301370:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763210.301376:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763210.301380:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763210.301386:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763210.301396:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787763210.301406:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763210.301418:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832602240/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763210.301437:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763210.301441:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763210.301450:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832602240/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763210.301466:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763210.301474:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763210.301498:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763210.301507:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763210.301519:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763210.301526:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763210.301530:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763210.301536:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763210.301550:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20924
00000400:00000200:0.0:1787763210.301568:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763210.301584:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763210.301594:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763210.301608:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763210.301631:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763210.301646:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763210.301657:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763210.301693:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763210.301700:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763210.301709:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763210.301724:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763210.301729:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763210.301737:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763210.301752:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763210.301758:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763210.301767:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763210.301775:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763210.301782:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763210.301792:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763210.301802:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763210.301809:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763210.301818:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763210.301826:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763210.301835:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763210.301842:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763210.301854:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763210.301870:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763210.301886:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763210.301899:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763210.301913:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763210.301920:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763210.301931:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763210.301942:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763210.301952:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763210.301968:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787763210.301979:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763210.301996:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832602368/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763210.302026:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763210.302034:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763210.302050:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832602368/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763210.302083:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763210.302105:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20935)
00000800:00000200:3.1:1787763210.302422:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763210.302427:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1787763210.302895:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763210.302898:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:3.1:1787763210.303287:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763210.303291:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000020:00000001:0.0:1787763211.964436:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763211.964441:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763211.964444:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763212.988396:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763212.988403:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763212.988408:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763214.012442:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763214.012451:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763214.012457:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763215.036490:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763215.036496:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763215.036501:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787763215.421184:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763215.421193:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787763215.422290:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763215.422296:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787763215.422726:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763215.422729:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787763215.423202:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763215.423206:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000020:00000001:0.0:1787763216.060411:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763216.060418:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763216.060423:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763217.084370:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763217.084375:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763217.084378:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763218.108393:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763218.108398:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763218.108401:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763219.132517:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763219.132526:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763219.132531:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763220.156467:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763220.156475:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763220.156479:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763220.540284:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763220.540298:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763220.540303:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763220.540309:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763220.540321:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763220.540333:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 20935
00000400:00000200:0.0:1787763220.540349:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763220.540363:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763220.540383:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763220.540403:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763220.540408:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763220.540410:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763220.540413:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763220.540417:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763220.540419:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763220.540422:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763220.540426:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763220.540428:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763220.540430:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763220.540433:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763220.540434:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763220.540436:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763220.540437:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763220.540439:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763220.540442:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763220.540444:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763220.540446:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763220.540451:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763220.540460:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763220.540464:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763220.540466:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763220.540476:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000020:00000040:0.0:1787763220.540480:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763220.540484:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763220.540486:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763220.540489:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763220.540493:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763220.540498:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763220.540507:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787763220.540515:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763220.540524:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832603008/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763220.540534:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763220.540538:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763220.540543:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832603008/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763220.540561:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763220.540564:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763220.540567:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763220.540569:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763220.540570:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763220.540573:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763220.540576:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20935
00000400:00000200:0.0:1787763220.540579:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763220.540582:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763220.540585:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763220.540587:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763220.540589:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763220.540591:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763220.540596:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763220.540613:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763220.540615:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763220.540619:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763220.540621:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763220.540624:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763220.540626:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763220.540629:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763220.540633:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763220.540636:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763220.540640:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763220.540644:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763220.540649:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763220.540654:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763220.540655:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763220.540663:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763220.540669:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763220.540671:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763220.540679:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763220.540684:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763220.540688:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763220.540691:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787763220.540695:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763220.540700:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763220.540704:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763220.540706:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763220.540709:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763220.540711:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763220.540717:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787763220.540724:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763220.540731:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832603136/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763220.540741:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763220.540745:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763220.540751:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832603136/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763220.540759:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763220.540764:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763220.540768:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763220.540771:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763220.540773:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763220.540779:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763220.540785:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20935
00000400:00000200:0.0:1787763220.540792:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763220.540799:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763220.540802:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763220.540807:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763220.540811:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763220.540812:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763220.540814:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763220.540816:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763220.540818:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763220.540823:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763220.540824:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763220.540826:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763220.540827:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763220.540831:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763220.540835:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763220.540838:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763220.540840:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763220.540842:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763220.540847:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763220.540851:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763220.540853:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763220.540856:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763220.540859:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763220.540862:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763220.540865:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763220.540870:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763220.540875:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763220.540881:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
00000020:00000040:0.0:1787763220.540886:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763220.540892:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763220.540893:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763220.540895:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763220.540898:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763220.540903:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763220.540909:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787763220.540912:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763220.540919:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832603264/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763220.540928:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763220.540930:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763220.540936:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832603264/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763220.540947:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763220.540950:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:3.1:1787763220.540954:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763220.540956:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:3.1:1787763220.540960:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000100:00080000:0.0:1787763220.540961:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763220.540967:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763220.540972:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763220.540976:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763220.540980:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763220.540985:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20935
00000400:00000200:0.0:1787763220.540990:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763220.540995:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763220.540998:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763220.541003:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763220.541006:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763220.541009:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763220.541012:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763220.541015:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763220.541018:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763220.541021:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763220.541024:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763220.541029:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763220.541043:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763220.541050:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763220.541053:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763220.541054:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763220.541055:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763220.541057:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763220.541058:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763220.541062:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763220.541064:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763220.541069:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763220.541071:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763220.541072:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763220.541076:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763220.541082:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763220.541083:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763220.541090:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763220.541095:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763220.541100:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763220.541101:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763220.541103:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763220.541104:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763220.541106:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763220.541110:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787763220.541114:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763220.541118:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832603392/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763220.541126:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763220.541128:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763220.541179:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832603392/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763220.541207:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763220.541215:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20945)
00000800:00000200:3.1:1787763220.541475:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763220.541477:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787763220.541730:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763220.541732:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787763220.541925:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763220.541927:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000020:00000001:0.0:1787763221.180371:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763221.180377:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763221.180381:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763222.204473:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763222.204480:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763222.204485:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763223.228392:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763223.228398:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763223.228403:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763224.252437:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763224.252443:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763224.252448:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763225.276426:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763225.276436:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763225.276443:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763225.660234:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763225.660247:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763225.660253:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763225.660259:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763225.660267:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763225.660281:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 20940
00000400:00000200:0.0:1787763225.660300:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763225.660317:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763225.660324:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763225.660335:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763225.660343:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763225.660346:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763225.660350:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763225.660355:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763225.660358:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763225.660361:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763225.660367:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763225.660370:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763225.660374:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763225.660378:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763225.660381:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763225.660383:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763225.660386:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763225.660389:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763225.660394:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763225.660397:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763225.660400:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763225.660404:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763225.660408:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763225.660410:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763225.660413:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763225.660424:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763225.660428:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763225.660434:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763225.660436:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763225.660440:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763225.660443:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763225.660448:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763225.660457:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763225.660466:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763225.660478:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832603520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763225.660492:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763225.660498:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763225.660504:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832603520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763225.660535:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763225.660542:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763225.660548:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763225.660552:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763225.660556:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763225.660580:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763225.660587:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20940
00000400:00000200:0.0:1787763225.660596:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763225.660606:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763225.660619:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763225.660634:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763225.660645:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763225.660655:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763225.660660:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763225.660664:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763225.660668:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763225.660673:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763225.660679:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763225.660690:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763225.660698:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763225.660709:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763225.660718:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763225.660722:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763225.660729:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763225.660736:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763225.660741:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763225.660752:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763225.660760:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763225.660770:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763225.660780:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763225.660788:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763225.660800:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763225.660811:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763225.660818:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763225.660829:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763225.660841:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763225.660850:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763225.660855:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763225.660860:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763225.660863:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763225.660868:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763225.660880:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763225.660890:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763225.660906:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832603648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763225.660949:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763225.660955:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763225.660970:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832603648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763225.661018:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763225.661024:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763225.661029:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763225.661030:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763225.661032:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763225.661036:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763225.661046:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20940
00000400:00000200:0.0:1787763225.661062:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763225.661074:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763225.661080:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763225.661089:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763225.661096:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763225.661105:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763225.661115:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763225.661122:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763225.661128:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763225.661169:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763225.661174:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763225.661181:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763225.661192:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763225.661204:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763225.661211:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763225.661219:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763225.661228:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763225.661236:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763225.661249:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763225.661254:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763225.661260:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763225.661268:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763225.661274:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763225.661284:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763225.661288:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763225.661296:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763225.661304:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763225.661312:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000010:0.0:1787763225.661313:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763225.661323:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763225.661328:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763225.661330:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763225.661333:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763225.661335:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763225.661337:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763225.661343:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763225.661348:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763225.661366:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832603776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763225.661384:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763225.661391:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763225.661400:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832603776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763225.661418:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763225.661423:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763225.661434:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763225.661440:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763225.661447:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763225.661455:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763225.661459:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763225.661471:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763225.661481:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20940
00000400:00000200:0.0:1787763225.661493:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763225.661507:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763225.661513:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763225.661526:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763225.661538:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763225.661552:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763225.661589:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763225.661600:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763225.661608:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763225.661618:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763225.661630:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763225.661639:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763225.661651:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763225.661665:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763225.661672:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763225.661681:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763225.661686:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763225.661697:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763225.661708:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763225.661715:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763225.661719:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763225.661728:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763225.661734:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763225.661743:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763225.661750:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763225.661762:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763225.661766:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763225.661783:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763225.661794:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763225.661804:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763225.661811:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763225.661820:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763225.661825:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763225.661831:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763225.661846:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763225.661858:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763225.661872:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832603904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763225.661894:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763225.661901:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763225.661916:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832603904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763225.661948:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763225.661962:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20950)
00000800:00000200:3.1:1787763225.662083:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763225.662422:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763225.662979:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787763226.300402:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763226.300409:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763226.300416:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763227.324376:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763227.324383:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763227.324388:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763228.348443:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763228.348450:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763228.348455:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763228.732526:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763228.732533:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763228.732538:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763228.732563:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763225/real 1787763228]  req@ffff9751f876ed00 x1874599832603520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763228.732597:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763228.732601:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763228.732606:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc234ec8.
00000400:00000200:0.0:1787763228.732613:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32ee0
00000400:00000010:0.0:1787763228.732619:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32ee0.
00000100:00000001:0.0:1787763228.732625:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763228.732636:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832603520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763228.732657:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832603520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763228.732685:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763228.732690:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763228.732695:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763228.732699:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763228.732709:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832603520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763228.732724:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763228.732735:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832603520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763228.732756:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763228.732762:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763228.732767:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763228.732771:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763228.732776:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763228.732780:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763228.732791:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763228.732801:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763228.732807:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763228.732818:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832603520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763228.732832:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832603520:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763228.732838:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763228.732841:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763228.732844:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832603520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763228.732858:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763228.732861:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763228.732864:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611800.
02000000:00000001:0.0:1787763228.732869:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763228.732872:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763228.732873:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787763228.732876:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763228.732879:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014800.
02000000:00000001:0.0:1787763228.732882:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763228.732884:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763228.732886:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:0.0:1787763228.732891:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763228.732893:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763228.732896:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763228.732898:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763228.732899:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763228.732901:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763228.732905:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763225/real 1787763228]  req@ffff9751f876e580 x1874599832603904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763228.732913:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763228.732915:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763228.732917:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2349a0.
00000400:00000200:0.0:1787763228.732920:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32000
00000400:00000010:0.0:1787763228.732922:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32000.
00000100:00000001:0.0:1787763228.732924:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763228.732928:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832603904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763228.732936:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832603904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763228.732943:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763228.732944:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763228.732946:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763228.732947:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763228.732952:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832603904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763228.732959:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763228.732964:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832603904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763228.732971:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763228.732974:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763228.732976:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763228.732977:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763228.732979:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763228.732980:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763228.732983:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763228.732988:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763228.732990:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763228.732994:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832603904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763228.733002:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832603904:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763228.733006:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763228.733010:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763225/real 1787763228]  req@ffff9751f876c000 x1874599832603648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763228.733018:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763228.733020:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763228.733022:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc234f78.
00000400:00000200:0.0:1787763228.733025:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32990
00000400:00000010:0.0:1787763228.733026:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32990.
00000100:00000001:0.0:1787763228.733029:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763228.733032:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832603648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763228.733046:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832603648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763228.733053:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763228.733054:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763228.733056:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763228.733058:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763228.733062:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832603648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763228.733069:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763228.733074:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832603648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763228.733081:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763228.733084:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763228.733086:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763228.733088:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763228.733089:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763228.733102:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763228.733105:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763228.733108:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763228.733111:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763228.733115:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832603648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763228.733123:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832603648:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763228.733128:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763228.733131:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763225/real 1787763228]  req@ffff9751f876e1c0 x1874599832603776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763228.733185:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763228.733187:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763228.733190:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc234160.
00000400:00000200:0.0:1787763228.733193:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32088
00000400:00000010:0.0:1787763228.733196:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32088.
00000100:00000001:0.0:1787763228.733200:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763228.733204:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832603776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763228.733215:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832603776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763228.733223:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763228.733225:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763228.733227:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763228.733229:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763228.733233:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832603776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763228.733239:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763228.733242:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832603776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763228.733246:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763228.733248:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763228.733250:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763228.733251:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763228.733253:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763228.733254:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763228.733258:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763228.733263:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763228.733266:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763228.733271:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832603776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763228.733279:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832603776:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763228.733283:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763228.733285:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763228.733289:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832603904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763228.733296:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763228.733298:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763228.733300:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610400.
02000000:00000001:0.0:1787763228.733303:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763228.733305:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763228.733306:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787763228.733309:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763228.733311:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016800.
02000000:00000001:0.0:1787763228.733314:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763228.733315:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763228.733317:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:0.0:1787763228.733320:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763228.733322:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763228.733323:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763228.733327:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832603648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763228.733335:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763228.733336:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763228.733338:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613c00.
02000000:00000001:0.0:1787763228.733342:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763228.733343:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763228.733344:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763228.733345:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763228.733346:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017000.
02000000:00000001:0.0:1787763228.733349:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763228.733350:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763228.733351:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:0.0:1787763228.733354:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763228.733355:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763228.733356:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763228.733358:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832603776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763250 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763228.733362:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763228.733363:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763228.733364:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613800.
02000000:00000001:0.0:1787763228.733366:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763228.733366:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763228.733367:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763228.733368:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763228.733369:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017800.
02000000:00000001:0.0:1787763228.733370:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763228.733371:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763228.733372:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:0.0:1787763228.733373:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763228.733374:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763228.733375:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763228.733381:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763228.733382:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763228.733383:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763228.733384:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763229.372395:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763229.372401:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763229.372407:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763230.396364:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763230.396371:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763230.396376:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763230.780291:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763230.780307:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763230.780312:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763230.780319:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763230.780328:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763230.780342:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 20945
00000400:00000200:0.0:1787763230.780358:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763230.780379:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763230.780389:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763230.780404:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763230.780416:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763230.780422:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763230.780427:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763230.780434:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763230.780438:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763230.780444:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763230.780450:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763230.780455:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763230.780460:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763230.780466:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763230.780470:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763230.780475:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763230.780479:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763230.780486:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763230.780493:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763230.780500:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763230.780505:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763230.780513:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763230.780519:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763230.780524:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763230.780557:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763230.780570:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763230.780578:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763230.780586:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763230.780590:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763230.780595:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763230.780600:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763230.780607:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763230.780620:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763230.780632:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763230.780660:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832604032/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763230.780681:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763230.780689:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763230.780701:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832604032/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763230.780752:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763230.780761:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763230.780767:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763230.780770:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763230.780774:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763230.780780:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763230.780787:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20945
00000400:00000200:0.0:1787763230.780796:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763230.780805:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763230.780811:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763230.780817:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763230.780822:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763230.780829:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763230.780839:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763230.780848:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763230.780856:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763230.780862:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763230.780871:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763230.780880:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763230.780892:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763230.780911:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763230.780926:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763230.780937:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763230.780951:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763230.780966:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763230.780982:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763230.780992:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763230.781000:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763230.781011:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763230.781019:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763230.781028:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763230.781041:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763230.781055:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763230.781063:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763230.781072:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763230.781081:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763230.781088:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763230.781095:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763230.781101:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763230.781107:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763230.781114:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763230.781126:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763230.781183:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763230.781198:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832604160/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763230.781224:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763230.781235:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763230.781257:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832604160/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763230.781294:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763230.781307:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763230.781324:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763230.781332:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763230.781336:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763230.781347:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763230.781363:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20945
00000400:00000200:0.0:1787763230.781382:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763230.781391:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763230.781399:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763230.781415:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763230.781426:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763230.781440:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763230.781448:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763230.781454:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763230.781459:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763230.781464:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763230.781473:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763230.781485:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763230.781492:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763230.781510:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763230.781521:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763230.781556:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763230.781565:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763230.781573:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763230.781597:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763230.781607:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763230.781616:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:3.1:1787763230.781624:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:0.0:1787763230.781627:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:3.1:1787763230.781634:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
02000000:00000001:0.0:1787763230.781636:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763230.781649:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763230.781658:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763230.781668:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763230.781678:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763230.781702:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763230.781715:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763230.781724:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763230.781732:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763230.781743:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763230.781748:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763230.781757:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763230.781769:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763230.781782:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763230.781804:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832604288/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763230.781822:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763230.781827:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763230.781834:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832604288/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763230.781850:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763230.781856:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763230.781863:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763230.781868:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763230.781874:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763230.781877:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763230.781881:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763230.781886:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763230.781891:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20945
00000400:00000200:0.0:1787763230.781900:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763230.781909:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763230.781915:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763230.781921:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763230.781927:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763230.781930:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763230.781934:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763230.781939:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763230.781941:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763230.781943:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763230.781946:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763230.781958:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763230.781961:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763230.781965:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763230.781968:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763230.781970:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763230.781973:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763230.781976:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763230.781979:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763230.781982:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763230.781984:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763230.781990:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763230.781995:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763230.781998:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763230.782003:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763230.782007:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763230.782011:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763230.782018:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763230.782022:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763230.782028:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763230.782030:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763230.782034:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763230.782037:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763230.782040:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763230.782048:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763230.782055:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763230.782066:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832604416/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763230.782080:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763230.782084:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763230.782091:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832604416/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763230.782105:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763230.782113:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20955)
00000800:00000200:3.1:1787763230.783171:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763230.783176:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787763230.783627:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763230.783631:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787763230.784059:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763230.784065:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:0.0:1787763231.420495:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763231.420502:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763231.420507:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763232.444436:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763232.444442:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763232.444447:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763233.468430:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763233.468437:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763233.468442:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763235.516322:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763235.516327:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763235.516331:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763235.900275:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763235.900292:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763235.900298:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763235.900310:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763235.900323:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763235.900340:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 20950
00000400:00000200:0.0:1787763235.900363:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763235.900377:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763235.900385:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763235.900397:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763235.900402:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763235.900406:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763235.900425:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763235.900432:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763235.900439:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763235.900443:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763235.900448:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763235.900451:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763235.900454:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763235.900459:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763235.900461:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763235.900464:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763235.900466:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763235.900471:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763235.900475:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763235.900478:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763235.900482:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763235.900486:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763235.900490:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763235.900517:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763235.900521:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763235.900533:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
00000020:00000040:0.0:1787763235.900540:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763235.900547:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763235.900550:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763235.900556:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763235.900560:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763235.900567:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763235.900581:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763235.900592:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763235.900609:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832604544/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763235.900630:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763235.900639:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763235.900650:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832604544/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763235.900692:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763235.900697:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763235.900702:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763235.900704:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763235.900706:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763235.900710:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763235.900714:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20950
00000400:00000200:0.0:1787763235.900720:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763235.900724:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763235.900728:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763235.900732:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763235.900735:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763235.900738:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763235.900741:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763235.900744:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763235.900746:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763235.900748:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763235.900752:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763235.900756:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763235.900759:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763235.900764:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763235.900767:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763235.900770:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763235.900774:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763235.900778:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763235.900787:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763235.900796:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763235.900802:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763235.900810:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763235.900821:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763235.900830:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763235.900839:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763235.900849:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763235.900857:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763235.900866:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000020:00000040:0.0:1787763235.900873:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763235.900882:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763235.900886:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763235.900892:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763235.900896:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763235.900901:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763235.900913:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787763235.900923:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763235.900937:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832604672/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763235.900959:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763235.900965:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763235.900974:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832604672/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763235.900996:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763235.901002:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763235.901013:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763235.901020:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763235.901025:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763235.901029:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763235.901035:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20950
00000400:00000200:0.0:1787763235.901044:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763235.901049:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763235.901055:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763235.901061:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763235.901072:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763235.901080:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763235.901087:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763235.901092:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763235.901096:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763235.901100:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763235.901105:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763235.901115:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763235.901119:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763235.901124:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763235.901126:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763235.901128:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763235.901132:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763235.901174:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763235.901178:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763235.901182:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763235.901189:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763235.901196:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763235.901204:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763235.901212:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763235.901218:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763235.901228:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763235.901234:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763235.901243:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000020:00000040:0.0:1787763235.901250:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763235.901256:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763235.901258:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763235.901269:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763235.901276:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763235.901284:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763235.901300:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763235.901317:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763235.901341:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832604800/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763235.901384:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763235.901396:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763235.901414:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832604800/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763235.901434:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763235.901441:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763235.901453:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763235.901458:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763235.901470:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763235.901475:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763235.901478:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:3.1:1787763235.901488:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763235.901489:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763235.901517:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20950
00000400:00000200:0.0:1787763235.901526:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:3.1:1787763235.901537:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000400:00000200:0.0:1787763235.901538:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763235.901545:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763235.901555:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763235.901563:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763235.901569:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763235.901577:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763235.901588:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763235.901592:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763235.901595:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763235.901599:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763235.901606:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763235.901613:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763235.901625:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763235.901634:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763235.901640:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763235.901644:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763235.901654:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763235.901665:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763235.901669:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763235.901676:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763235.901681:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763235.901686:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763235.901695:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763235.901702:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763235.901712:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763235.901716:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763235.901723:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763235.901731:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763235.901742:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763235.901747:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763235.901756:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763235.901761:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763235.901765:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763235.901774:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763235.901782:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763235.901799:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832604928/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763235.901818:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763235.901824:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763235.901831:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832604928/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763235.901847:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763235.901859:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20960)
00000800:00000200:3.1:1787763235.902384:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763235.902388:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787763235.902788:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763235.902792:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787763235.903300:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763235.903304:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:0.0:1787763236.540410:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763236.540416:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763236.540422:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763238.588487:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763238.588494:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763238.588498:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763239.612293:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763239.612297:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763239.612300:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787763241.020947:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763241.020955:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787763241.021546:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763241.021548:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787763241.021770:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763241.021772:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787763241.021961:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763241.021963:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:0.0:1787763241.660511:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763241.660518:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763241.660522:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763242.684395:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763242.684401:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763242.684406:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763244.732403:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763244.732409:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763244.732414:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763245.756351:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763245.756355:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763245.756358:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763246.140343:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763246.140357:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763246.140363:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763246.140369:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763246.140376:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763246.140388:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 20960
00000400:00000200:0.0:1787763246.140401:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763246.140411:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763246.140417:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763246.140448:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763246.140454:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763246.140459:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763246.140466:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763246.140473:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763246.140477:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763246.140482:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763246.140488:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763246.140493:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763246.140497:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763246.140503:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763246.140507:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763246.140511:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763246.140515:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763246.140521:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763246.140528:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763246.140534:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763246.140538:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763246.140545:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763246.140551:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763246.140555:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763246.140559:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763246.140574:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763246.140581:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763246.140587:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763246.140590:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763246.140595:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763246.140600:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763246.140606:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763246.140619:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787763246.140630:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763246.140645:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832605568/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763246.140665:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763246.140672:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763246.140682:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832605568/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763246.140719:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763246.140725:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763246.140732:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763246.140734:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763246.140738:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763246.140745:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763246.140752:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20960
00000400:00000200:0.0:1787763246.140763:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763246.140774:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763246.140784:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763246.140801:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763246.140815:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763246.140826:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763246.140835:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763246.140847:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763246.140854:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763246.140864:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763246.140873:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763246.140885:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763246.140898:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763246.140914:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763246.140928:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763246.140956:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763246.140965:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763246.140978:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763246.140997:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763246.141013:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763246.141024:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763246.141039:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763246.141053:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763246.141062:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763246.141073:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763246.141086:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763246.141097:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763246.141113:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763246.141126:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000800:00000200:3.1:1787763246.141231:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:0.0:1787763246.141235:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:3.1:1787763246.141238:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
02000000:00000001:0.0:1787763246.141242:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763246.141253:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763246.141259:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763246.141268:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763246.141286:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787763246.141301:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763246.141325:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832605696/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763246.141359:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763246.141369:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763246.141392:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832605696/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763246.141456:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763246.141467:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763246.141480:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763246.141488:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763246.141498:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763246.141512:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763246.141525:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20960
00000400:00000200:0.0:1787763246.141544:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763246.141563:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763246.141575:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763246.141595:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763246.141609:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763246.141620:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763246.141633:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763246.141644:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763246.141651:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763246.141661:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763246.141670:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763246.141683:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763246.141693:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763246.141712:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763246.141723:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763246.141737:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763246.141751:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763246.141766:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763246.141775:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763246.141784:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763246.141791:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763246.141799:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763246.141810:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763246.141817:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763246.141824:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763246.141834:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763246.141840:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763246.141851:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763246.141859:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763246.141866:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763246.141873:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763246.141883:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763246.141891:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763246.141899:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763246.141915:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787763246.141933:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763246.141957:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832605824/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763246.141991:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763246.142002:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763246.142014:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832605824/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763246.142052:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763246.142062:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763246.142072:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763246.142086:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763246.142106:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763246.142115:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763246.142127:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763246.142176:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763246.142189:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 20960
00000400:00000200:0.0:1787763246.142209:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763246.142223:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763246.142237:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763246.142254:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763246.142267:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763246.142281:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763246.142294:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763246.142310:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763246.142319:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763246.142330:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763246.142340:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763246.142354:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763246.142368:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763246.142387:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763246.142400:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763246.142415:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763246.142453:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763246.142465:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763246.142481:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000800:00000200:3.1:1787763246.142492:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763246.142495:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763246.142499:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
02000000:00000001:0.0:1787763246.142504:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763246.142516:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763246.142528:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763246.142538:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763246.142546:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763246.142560:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763246.142570:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763246.142584:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763246.142596:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763246.142606:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763246.142613:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763246.142623:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763246.142630:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763246.142638:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763246.142652:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787763246.142666:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763246.142691:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832605952/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763246.142731:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763246.142742:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763246.142764:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832605952/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763246.142824:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763246.142838:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (20970)
00000800:00000200:3.1:1787763246.143178:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763246.143181:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787763246.143565:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763246.143568:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:0.0:1787763247.804398:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763247.804404:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763247.804435:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763250.876443:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763250.876448:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763250.876452:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787763251.261347:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763251.261358:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787763251.262369:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763251.262373:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787763251.262756:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763251.262761:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787763251.263260:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763251.263264:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000020:00000001:0.0:1787763253.948425:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763253.948431:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763253.948436:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763254.972349:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763254.972354:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763254.972357:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763256.380277:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763256.380286:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763256.380291:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763256.380300:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763256.380306:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763256.380311:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763256.380315:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763256.380321:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763256.380327:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763256.381023:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763256.381033:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787763256.382024:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763256.382025:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787763256.382253:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763256.382255:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787763256.382468:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763256.382470:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:0.0:1787763257.020331:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763257.020339:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763257.020356:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763257.404240:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763257.404253:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763257.404256:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763257.404265:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763257.404272:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763257.404277:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763257.404280:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763257.404284:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763257.404289:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763258.044320:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763258.044324:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763258.044327:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763258.428256:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763258.428264:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763258.428267:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763258.428273:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763258.428277:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763258.428281:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763258.428284:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763258.428287:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763258.428290:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763259.452242:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763259.452254:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763259.452256:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763259.452263:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763259.452268:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763259.452274:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763259.452278:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763259.452283:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763259.452288:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763260.092309:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763260.092322:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763260.092339:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763260.476208:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763260.476217:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763260.476220:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763260.476226:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763260.476231:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763260.476235:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763260.476239:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763260.476242:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763260.476245:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763261.116406:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763261.116412:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763261.116416:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763261.500263:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763261.500274:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763261.500279:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763261.500289:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763261.500297:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763261.500305:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763261.500310:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763261.500334:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763261.500341:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763261.501609:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763261.501616:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787763261.502413:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763261.502418:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787763261.502986:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763261.502992:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787763261.503619:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763261.503624:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:0.0:1787763262.524246:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763262.524257:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763262.524260:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763262.524267:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763262.524274:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763262.524279:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763262.524283:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763262.524287:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763262.524291:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763263.164411:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763263.164414:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763263.164418:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763263.548291:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763263.548338:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763263.548344:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763263.548353:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763263.548362:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763263.548369:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763263.548375:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763263.548380:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763263.548385:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763264.188309:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763264.188313:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763264.188316:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763264.572207:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763264.572214:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763264.572215:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763264.572220:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763264.572224:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763264.572227:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763264.572229:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763264.572232:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763264.572234:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763265.596343:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763265.596355:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763265.596357:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763265.596366:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763265.596371:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763265.596376:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763265.596379:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763265.596384:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763265.596389:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763266.620240:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763266.620256:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763266.620259:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763266.620267:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763266.620273:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763266.620291:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.620295:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763266.620300:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.620305:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763266.620813:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763266.620829:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763266.620840:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763266.620875:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bed00 x1874599832607616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.620946:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763266.620953:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763266.620966:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763266.620977:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763266.620998:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763266.621017:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832607616:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763266.621042:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763266.621054:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763266.621063:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763266.621073:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763266.621090:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763266.621097:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763266.621111:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763266.621123:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763266.621177:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763266.621186:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832607616, portal 25
00000100:00000001:0.0:1787763266.621190:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763266.621193:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049857280 : -115096659694336 : ffff9751f87bed00)
00000100:00000040:0.0:1787763266.621205:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bed00 x1874599832607616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763291 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.621217:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763266.621223:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763266.621228:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763266.621235:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832607616, offset 0
00000400:00000200:0.0:1787763266.621246:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763266.621261:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763266.621273:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763266.621306:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763266.621317:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763266.621323:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763266.621328:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763266.621334:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57982:57982:256:4294967295] 192.168.203.110@tcp LPNI seq info [57982:57982:8:4294967295]
00000400:00000200:0.0:1787763266.621341:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763266.621360:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763266.621370:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763266.621379:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763266.621391:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763266.621398:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787763266.621408:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763266.621413:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1787763266.621420:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763266.621424:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763266.621431:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763266.621437:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763266.621443:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763266.621446:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763266.621451:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763266.621457:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bed00 x1874599832607616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763291 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.621490:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763266.621498:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bed00 x1874599832607616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763291 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.621508:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763266.621511:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763266.621530:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763266.621540:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763266.621544:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315654900
00000100:00000010:0.0:1787763266.621551:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940cc00.
00000100:00000001:0.0:1787763266.621559:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763266.621571:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315654900
00000100:00000010:0.0:1787763266.621576:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940cba0.
00000100:00000001:0.0:1787763266.621582:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787763266.621588:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763266.621593:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763266.621599:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763266.621611:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763266/real 1787763266]  req@ffff9751f87bed00 x1874599832607616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763266.621630:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763266.621634:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763266.621638:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763266.621647:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763266.621651:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763266.621658:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763266.621671:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bed00 x1874599832607616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763266.621701:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bed00 x1874599832607616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.621785:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763266.621796:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.621814:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763266.621822:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763266.621848:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bed00 x1874599832607616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.621883:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763266.621895:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bed00 x1874599832607616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.621916:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763266.621924:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.621931:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763266.621934:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.621950:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763266.621954:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763266.621966:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763266.621982:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763266.621991:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763266.622005:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bed00 x1874599832607616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763266.622036:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832607616:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763266.622046:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.622051:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763266.622062:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bed00 x1874599832607616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.622085:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763266.622092:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763266.622097:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763266.622104:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763266.622111:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763266.622118:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787763266.622129:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763266.622182:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612000.
02000000:00000001:0.0:1787763266.622193:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763266.622197:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763266.622202:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bed00.
00000100:00000001:0.0:1787763266.622215:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763266.622221:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.622228:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.622253:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763266.622259:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763266.622271:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763266.622279:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763266.622315:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763266.622328:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bde00 x1874599832607744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.622357:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763266.622365:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763266.622376:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763266.622380:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763266.622388:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763266.622398:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832607744:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763266.622408:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763266.622417:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763266.622422:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763266.622433:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763266.622440:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763266.622445:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763266.622457:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763266.622463:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763266.622470:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763266.622480:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832607744, portal 10
00000100:00000001:0.0:1787763266.622489:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763266.622496:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049853440 : -115096659698176 : ffff9751f87bde00)
00000100:00000040:0.0:1787763266.622517:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bde00 x1874599832607744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763291 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.622546:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763266.622557:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763266.622565:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763266.622572:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832607744, offset 0
00000400:00000200:0.0:1787763266.622591:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763266.622608:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763266.622624:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763266.622636:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763266.622652:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763266.622665:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763266.622677:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763266.622693:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57983:57983:256:4294967295] 192.168.203.110@tcp LPNI seq info [57983:57983:8:4294967295]
00000400:00000200:0.0:1787763266.622708:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763266.622736:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763266.622747:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763266.622758:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763266.622781:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763266.622789:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787763266.622816:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763266.622823:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1787763266.622836:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763266.622841:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763266.622848:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763266.622860:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763266.622868:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763266.622874:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763266.622885:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763266.622897:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bde00 x1874599832607744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763291 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.622922:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763266.622931:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bde00 x1874599832607744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763291 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.622948:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763266.622951:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763266.622957:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763266.622962:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:0.0:1787763266.622966:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763266.622969:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315653665
00000100:00000001:0.0:1787763266.622973:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315654900
00000100:00000001:0.0:1787763266.622976:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315653665
00000100:00000001:0.0:1787763266.622979:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315654900
00000100:00000001:0.0:1787763266.622982:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763266.622984:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763266.622987:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763266.622991:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763266/real 1787763266]  req@ffff9751f87bde00 x1874599832607744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763266.623002:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763266.623004:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763266.623006:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763266.623010:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763266.623012:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763266.623015:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763266.623019:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bde00 x1874599832607744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763266.623029:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bde00 x1874599832607744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.623037:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763266.623039:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.623041:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763266.623043:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763266.623047:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bde00 x1874599832607744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.623056:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763266.623061:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bde00 x1874599832607744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.623070:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763266.623073:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.623075:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763266.623076:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.623079:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763266.623092:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763266.623095:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763266.623099:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763266.623102:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763266.623106:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bde00 x1874599832607744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763266.623116:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832607744:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763266.623121:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763266.623126:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87be1c0 x1874599832607872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.623189:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763266.623192:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763266.623196:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763266.623200:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763266.623204:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763266.623209:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832607872:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763266.623214:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763266.623217:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763266.623220:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763266.623223:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763266.623228:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763266.623230:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763266.623235:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763266.623238:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763266.623243:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763266.623246:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832607872, portal 10
00000100:00000001:0.0:1787763266.623249:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763266.623251:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049854400 : -115096659697216 : ffff9751f87be1c0)
00000100:00000040:0.0:1787763266.623258:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87be1c0 x1874599832607872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763291 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.623267:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763266.623270:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763266.623272:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763266.623275:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832607872, offset 0
00000400:00000200:0.0:1787763266.623307:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763266.623314:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763266.623320:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763266.623324:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763266.623327:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763266.623331:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763266.623334:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763266.623338:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57984:57984:256:4294967295] 192.168.203.110@tcp LPNI seq info [57984:57984:8:4294967295]
00000400:00000200:0.0:1787763266.623344:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763266.623352:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763266.623358:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763266.623363:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763266.623368:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763266.623372:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787763266.623378:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763266.623381:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1787763266.623386:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763266.623390:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763266.623394:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763266.623399:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763266.623403:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763266.623405:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763266.623408:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763266.623414:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87be1c0 x1874599832607872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763291 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.623424:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763266.623428:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87be1c0 x1874599832607872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763291 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.623437:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763266.623439:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763266.623444:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763266.623448:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:0.0:1787763266.623452:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763266.623455:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315653654
00000100:00000001:0.0:1787763266.623458:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315654900
00000100:00000001:0.0:1787763266.623461:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315653654
00000100:00000001:0.0:1787763266.623463:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315654900
00000100:00000001:0.0:1787763266.623466:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763266.623468:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763266.623471:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763266.623475:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763266/real 1787763266]  req@ffff9751f87be1c0 x1874599832607872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763266.623484:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763266.623486:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763266.623488:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763266.623492:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763266.623494:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763266.623497:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763266.623501:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87be1c0 x1874599832607872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763266.623511:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87be1c0 x1874599832607872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.623519:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763266.623521:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.623523:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763266.623525:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763266.623529:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87be1c0 x1874599832607872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.623537:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763266.623541:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87be1c0 x1874599832607872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.623549:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763266.623552:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.623554:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763266.623556:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.623558:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763266.623560:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763266.623563:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763266.623567:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763266.623570:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763266.623575:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87be1c0 x1874599832607872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763266.623592:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832607872:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763266.623600:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.623605:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763266.623611:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bde00 x1874599832607744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.623625:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763266.623628:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763266.623632:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763266.623638:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763266.623641:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763266.623645:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763266.623649:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763266.623653:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610c00.
02000000:00000001:0.0:1787763266.623658:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763266.623661:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763266.623664:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bde00.
00000100:00000001:0.0:1787763266.623669:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763266.623672:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.623674:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763266.623680:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87be1c0 x1874599832607872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.623688:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763266.623690:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763266.623692:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763266.623696:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763266.623697:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763266.623699:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763266.623701:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763266.623703:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610800.
02000000:00000001:0.0:1787763266.623705:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763266.623707:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763266.623709:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87be1c0.
00000100:00000001:0.0:1787763266.623711:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763266.623713:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.623715:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.623721:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763266.623722:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763266.623725:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763266.623728:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763266.623730:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763266.623735:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc000 x1874599832608000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.623745:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763266.623747:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763266.623750:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763266.623752:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763266.623755:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763266.623758:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832608000:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763266.623762:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763266.623765:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763266.623767:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763266.623770:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763266.623774:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763266.623777:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763266.623782:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763266.623785:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763266.623789:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763266.623794:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832608000, portal 4
00000100:00000001:0.0:1787763266.623796:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763266.623798:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049845760 : -115096659705856 : ffff9751f87bc000)
00000100:00000040:0.0:1787763266.623805:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc000 x1874599832608000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763291 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.623814:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763266.623817:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763266.623819:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763266.623822:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832608000, offset 0
00000400:00000200:0.0:1787763266.623827:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763266.623832:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763266.623837:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763266.623841:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763266.623844:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763266.623848:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763266.623851:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763266.623855:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57985:57985:256:4294967295] 192.168.203.110@tcp LPNI seq info [57985:57985:8:4294967295]
00000400:00000200:0.0:1787763266.623861:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763266.623869:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763266.623874:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763266.623879:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763266.623884:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763266.623887:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:0.1:1787763266.623892:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763266.623896:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:0.0:1787763266.623902:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763266.623904:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763266.623908:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763266.623913:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763266.623916:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763266.623919:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763266.623922:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763266.623926:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc000 x1874599832608000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763291 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.623936:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763266.623940:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc000 x1874599832608000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763291 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.623948:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763266.623950:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763266.623955:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763266.623958:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:0.0:1787763266.623962:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763266.623965:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315653671
00000100:00000001:0.0:1787763266.623968:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315654900
00000100:00000001:0.0:1787763266.623971:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315653671
00000100:00000001:0.0:1787763266.623975:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315654900
00000100:00000001:0.0:1787763266.623978:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763266.623981:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763266.623983:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763266.623988:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763266/real 1787763266]  req@ffff9751f87bc000 x1874599832608000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763266.623997:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763266.623998:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763266.624000:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763266.624003:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763266.624005:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763266.624008:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763266.624011:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc000 x1874599832608000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763266.624022:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc000 x1874599832608000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.624030:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763266.624032:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.624034:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763266.624036:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763266.624040:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc000 x1874599832608000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.624048:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763266.624052:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc000 x1874599832608000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.624060:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763266.624063:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.624065:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763266.624066:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.624069:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763266.624072:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763266.624075:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763266.624080:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763266.624083:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763266.624087:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc000 x1874599832608000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763266.624096:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832608000:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763266.624101:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.624103:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763266.624106:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc000 x1874599832608000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763291 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763266.624114:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763266.624116:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763266.624118:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763266.624121:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763266.624123:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763266.624125:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787763266.624127:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763266.624129:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611c00.
02000000:00000001:0.0:1787763266.624177:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763266.624180:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763266.624185:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc000.
00000100:00000001:0.0:1787763266.624190:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763266.624194:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.624198:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763266.624203:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763266.624205:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763266.624207:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763266.624210:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763267.260342:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763267.260347:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763267.260351:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763267.644297:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763267.644307:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763267.644310:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763267.644317:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763267.644322:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763267.644326:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763267.644329:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763267.644333:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763267.644336:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763268.668343:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763268.668354:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763268.668358:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763268.668366:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763268.668371:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763268.668376:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763268.668380:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763268.668383:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763268.668387:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763269.692276:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763269.692287:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763269.692289:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763269.692296:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763269.692300:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763269.692306:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763269.692309:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763269.692312:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763269.692316:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763270.332426:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763270.332433:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763270.332449:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763270.716319:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763270.716329:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763270.716332:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763270.716339:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763270.716345:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763270.716350:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763270.716355:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763270.716360:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763270.716363:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763271.740283:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763271.740290:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763271.740295:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763271.740302:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763271.740311:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763271.740321:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.740333:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763271.740346:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.740353:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763271.740520:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763271.740525:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763271.740527:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763271.740543:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7931680 x1874599832608128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.740575:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763271.740578:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763271.740590:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763271.740594:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763271.740602:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763271.740615:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832608128:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763271.740629:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763271.740635:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763271.740639:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763271.740654:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763271.740666:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763271.740676:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763271.740690:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763271.740696:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763271.740705:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763271.740727:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832608128, portal 25
00000100:00000001:0.0:1787763271.740732:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763271.740736:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497727104 : -115097211824512 : ffff9751d7931680)
00000100:00000040:0.0:1787763271.740750:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7931680 x1874599832608128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763296 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.740772:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763271.740777:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763271.740792:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763271.740797:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832608128, offset 0
00000400:00000200:0.0:1787763271.740808:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763271.740819:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763271.740829:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763271.740840:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763271.740853:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763271.740866:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763271.740872:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763271.740881:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57986:57986:256:4294967295] 192.168.203.110@tcp LPNI seq info [57986:57986:8:4294967295]
00000400:00000200:0.0:1787763271.740907:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763271.740936:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763271.740960:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763271.740970:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763271.740987:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763271.740994:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787763271.741015:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763271.741027:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1787763271.741046:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763271.741051:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763271.741063:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763271.741077:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763271.741088:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763271.741104:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763271.741115:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763271.741124:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7931680 x1874599832608128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763296 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.741180:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763271.741193:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7931680 x1874599832608128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763296 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.741225:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763271.741230:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763271.741241:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763271.741264:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763271.741269:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315660020
00000100:00000010:0.0:1787763271.741294:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940cf00.
00000100:00000001:0.0:1787763271.741307:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763271.741323:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315660020
00000100:00000010:0.0:1787763271.741333:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940cc00.
00000100:00000001:0.0:1787763271.741340:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787763271.741351:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763271.741358:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763271.741363:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763271.741375:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763271/real 1787763271]  req@ffff9751d7931680 x1874599832608128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763271.741398:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763271.741401:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763271.741406:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763271.741420:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763271.741430:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763271.741446:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763271.741463:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7931680 x1874599832608128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763271.741490:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7931680 x1874599832608128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.741503:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763271.741505:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.741510:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763271.741513:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763271.741524:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7931680 x1874599832608128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.741537:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763271.741549:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7931680 x1874599832608128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.741575:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763271.741580:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.741581:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763271.741581:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.741594:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763271.741596:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763271.741614:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763271.741627:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763271.741633:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763271.741645:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7931680 x1874599832608128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763271.741661:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832608128:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763271.741674:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.741691:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763271.741707:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7931680 x1874599832608128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.741721:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763271.741727:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763271.741731:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763271.741737:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763271.741744:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763271.741748:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787763271.741758:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763271.741767:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8800.
02000000:00000001:0.0:1787763271.741784:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763271.741787:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763271.741789:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7931680.
00000100:00000001:0.0:1787763271.741795:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763271.741797:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.741800:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.741809:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763271.741811:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763271.741817:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763271.741822:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763271.741824:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763271.741831:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930b40 x1874599832608256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.741854:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763271.741857:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763271.741862:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763271.741865:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763271.741868:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763271.741873:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832608256:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763271.741876:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763271.741879:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763271.741881:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763271.741883:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763271.741887:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763271.741889:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763271.741894:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763271.741896:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763271.741899:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763271.741902:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832608256, portal 10
00000100:00000001:0.0:1787763271.741903:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763271.741904:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497724224 : -115097211827392 : ffff9751d7930b40)
00000100:00000040:0.0:1787763271.741910:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930b40 x1874599832608256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763296 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.741915:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763271.741917:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763271.741920:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763271.741921:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832608256, offset 0
00000400:00000200:0.0:1787763271.741924:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763271.741928:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763271.741931:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763271.741933:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763271.741935:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763271.741937:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763271.741939:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763271.741941:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57987:57987:256:4294967295] 192.168.203.110@tcp LPNI seq info [57987:57987:8:4294967295]
00000400:00000200:0.0:1787763271.741944:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763271.741950:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763271.741953:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763271.741955:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763271.741958:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763271.741960:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787763271.741963:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763271.741966:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1787763271.741969:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763271.741970:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763271.741972:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763271.741975:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763271.741977:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763271.741979:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763271.741980:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763271.741983:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930b40 x1874599832608256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763296 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.741989:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763271.741991:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930b40 x1874599832608256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763296 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.741996:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763271.741997:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763271.742000:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763271.742003:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:0.0:1787763271.742006:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763271.742007:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315658784
00000100:00000001:0.0:1787763271.742010:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315660020
00000100:00000001:0.0:1787763271.742011:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315658784
00000100:00000001:0.0:1787763271.742013:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315660020
00000100:00000001:0.0:1787763271.742014:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763271.742015:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763271.742017:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763271.742019:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763271/real 1787763271]  req@ffff9751d7930b40 x1874599832608256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763271.742024:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763271.742025:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763271.742027:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763271.742029:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763271.742030:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763271.742031:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763271.742033:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930b40 x1874599832608256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763271.742039:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930b40 x1874599832608256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.742043:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763271.742044:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.742045:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763271.742046:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763271.742048:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930b40 x1874599832608256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.742052:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763271.742054:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930b40 x1874599832608256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.742060:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763271.742061:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.742063:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763271.742063:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.742065:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763271.742076:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763271.742077:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763271.742079:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763271.742081:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763271.742083:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930b40 x1874599832608256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763271.742089:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832608256:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763271.742092:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763271.742094:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933480 x1874599832608384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.742108:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763271.742109:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763271.742110:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763271.742112:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763271.742113:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763271.742114:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832608384:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763271.742116:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763271.742118:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763271.742119:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763271.742120:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763271.742122:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763271.742123:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763271.742126:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763271.742127:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763271.742129:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763271.742151:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832608384, portal 10
00000100:00000001:0.0:1787763271.742153:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763271.742154:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497734784 : -115097211816832 : ffff9751d7933480)
00000100:00000040:0.0:1787763271.742158:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933480 x1874599832608384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763296 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.742174:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763271.742175:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763271.742177:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763271.742179:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832608384, offset 0
00000400:00000200:0.0:1787763271.742182:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763271.742185:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763271.742187:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763271.742189:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763271.742191:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763271.742193:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763271.742195:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763271.742197:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57988:57988:256:4294967295] 192.168.203.110@tcp LPNI seq info [57988:57988:8:4294967295]
00000400:00000200:0.0:1787763271.742200:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763271.742204:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763271.742206:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763271.742209:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763271.742211:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763271.742213:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787763271.742216:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763271.742218:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1787763271.742221:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763271.742222:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763271.742224:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763271.742227:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763271.742229:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763271.742231:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763271.742232:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763271.742235:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933480 x1874599832608384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763296 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.742240:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763271.742242:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933480 x1874599832608384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763296 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.742259:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763271.742260:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763271.742263:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763271.742266:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:0.0:1787763271.742268:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763271.742270:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315658773
00000100:00000001:0.0:1787763271.742272:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315660020
00000100:00000001:0.0:1787763271.742274:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315658773
00000100:00000001:0.0:1787763271.742275:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315660020
00000100:00000001:0.0:1787763271.742276:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763271.742278:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763271.742279:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763271.742282:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763271/real 1787763271]  req@ffff9751d7933480 x1874599832608384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763271.742287:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763271.742288:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763271.742289:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763271.742291:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763271.742293:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763271.742294:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763271.742296:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933480 x1874599832608384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763271.742312:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933480 x1874599832608384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.742318:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763271.742319:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.742320:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763271.742321:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763271.742323:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933480 x1874599832608384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.742327:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763271.742329:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933480 x1874599832608384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.742333:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763271.742335:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.742336:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763271.742336:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.742338:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763271.742339:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763271.742340:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763271.742342:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763271.742343:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763271.742346:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933480 x1874599832608384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763271.742351:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832608384:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763271.742353:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.742355:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763271.742357:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930b40 x1874599832608256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.742361:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763271.742363:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763271.742364:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763271.742366:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763271.742367:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763271.742368:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763271.742370:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763271.742371:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca400.
02000000:00000001:0.0:1787763271.742373:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763271.742374:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763271.742376:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930b40.
00000100:00000001:0.0:1787763271.742377:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763271.742378:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.742380:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763271.742382:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933480 x1874599832608384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.742385:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763271.742386:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763271.742387:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763271.742388:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763271.742389:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763271.742390:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763271.742391:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763271.742392:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb800.
02000000:00000001:0.0:1787763271.742393:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763271.742394:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763271.742395:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933480.
00000100:00000001:0.0:1787763271.742398:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763271.742398:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.742399:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.742402:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763271.742403:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763271.742404:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763271.742405:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763271.742406:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763271.742409:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930000 x1874599832608512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.742413:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763271.742414:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763271.742416:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763271.742417:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763271.742418:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763271.742420:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832608512:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763271.742422:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763271.742424:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763271.742425:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763271.742426:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763271.742428:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763271.742429:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763271.742432:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763271.742433:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763271.742435:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763271.742437:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832608512, portal 4
00000100:00000001:0.0:1787763271.742439:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763271.742440:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497721344 : -115097211830272 : ffff9751d7930000)
00000100:00000040:0.0:1787763271.742444:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930000 x1874599832608512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763296 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.742449:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763271.742451:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763271.742452:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763271.742454:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832608512, offset 0
00000400:00000200:0.0:1787763271.742456:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763271.742458:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763271.742460:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763271.742469:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763271.742470:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763271.742472:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763271.742474:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763271.742476:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57989:57989:256:4294967295] 192.168.203.110@tcp LPNI seq info [57989:57989:8:4294967295]
00000400:00000200:0.0:1787763271.742478:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763271.742483:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763271.742487:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763271.742489:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763271.742491:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763271.742493:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787763271.742496:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763271.742498:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1787763271.742501:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763271.742502:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763271.742504:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763271.742507:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763271.742509:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763271.742510:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763271.742511:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763271.742514:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930000 x1874599832608512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763296 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.742519:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763271.742521:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930000 x1874599832608512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763296 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.742525:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763271.742526:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763271.742527:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763271.742529:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:0.0:1787763271.742532:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763271.742533:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315658790
00000100:00000001:0.0:1787763271.742535:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315660020
00000100:00000001:0.0:1787763271.742536:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315658790
00000100:00000001:0.0:1787763271.742538:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315660020
00000100:00000001:0.0:1787763271.742539:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763271.742540:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763271.742541:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763271.742543:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763271/real 1787763271]  req@ffff9751d7930000 x1874599832608512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763271.742549:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763271.742550:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763271.742551:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763271.742553:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763271.742554:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763271.742555:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763271.742557:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930000 x1874599832608512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763271.742563:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930000 x1874599832608512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.742567:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763271.742568:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.742569:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763271.742570:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763271.742572:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930000 x1874599832608512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.742576:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763271.742578:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930000 x1874599832608512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.742582:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763271.742584:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.742585:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763271.742585:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.742586:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763271.742587:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763271.742589:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763271.742591:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763271.742604:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763271.742607:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930000 x1874599832608512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763271.742612:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832608512:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763271.742615:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.742616:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763271.742618:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930000 x1874599832608512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763296 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763271.742621:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763271.742622:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763271.742623:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763271.742624:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763271.742625:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763271.742626:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787763271.742627:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763271.742628:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8c00.
02000000:00000001:0.0:1787763271.742629:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763271.742630:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763271.742631:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930000.
00000100:00000001:0.0:1787763271.742632:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763271.742633:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.742634:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763271.742636:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763271.742637:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763271.742638:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763271.742640:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763272.764311:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763272.764321:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763272.764324:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763272.764330:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763272.764335:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763272.764339:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763272.764343:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763272.764346:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763272.764349:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763273.404487:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763273.404496:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763273.404502:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763273.788268:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763273.788277:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763273.788280:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763273.788287:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763273.788292:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763273.788297:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763273.788301:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763273.788305:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763273.788309:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763274.812306:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763274.812318:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763274.812321:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763274.812331:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763274.812338:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763274.812344:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763274.812349:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763274.812354:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763274.812359:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763275.836214:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763275.836247:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763275.836267:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763275.836274:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763275.836279:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763275.836284:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763275.836288:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763275.836291:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763275.836295:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763276.476307:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763276.476311:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763276.476314:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763276.860251:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763276.860257:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763276.860260:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763276.860267:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763276.860272:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763276.860278:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.860284:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763276.860289:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.860296:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763276.860449:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763276.860456:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763276.860461:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763276.860486:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930000 x1874599832608640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.860505:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763276.860510:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763276.860528:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763276.860534:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763276.860540:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763276.860559:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832608640:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763276.860577:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763276.860584:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763276.860590:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763276.860596:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763276.860606:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763276.860613:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763276.860626:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763276.860635:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763276.860646:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763276.860657:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832608640, portal 25
00000100:00000001:0.0:1787763276.860664:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763276.860668:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497721344 : -115097211830272 : ffff9751d7930000)
00000100:00000040:0.0:1787763276.860680:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930000 x1874599832608640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763301 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.860710:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763276.860715:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763276.860721:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763276.860729:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832608640, offset 0
00000400:00000200:0.0:1787763276.860742:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763276.860769:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763276.860780:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763276.860789:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763276.860795:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763276.860799:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763276.860802:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763276.860806:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57990:57990:256:4294967295] 192.168.203.110@tcp LPNI seq info [57990:57990:8:4294967295]
00000400:00000200:0.0:1787763276.860811:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763276.860840:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763276.860853:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763276.860862:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763276.860876:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763276.860884:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787763276.860898:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763276.860906:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1787763276.860918:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763276.860922:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763276.860930:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763276.860941:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763276.860951:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763276.860955:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763276.860963:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763276.860973:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930000 x1874599832608640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763301 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.861008:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763276.861015:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930000 x1874599832608640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763301 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.861024:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763276.861027:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763276.861036:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763276.861040:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763276.861043:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315665139
00000100:00000010:0.0:1787763276.861048:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940cba0.
00000100:00000001:0.0:1787763276.861055:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763276.861062:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315665139
00000100:00000010:0.0:1787763276.861065:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940cea0.
00000100:00000001:0.0:1787763276.861068:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787763276.861072:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763276.861074:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763276.861077:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763276.861082:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763276/real 1787763276]  req@ffff9751d7930000 x1874599832608640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763276.861092:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763276.861093:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763276.861096:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763276.861100:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763276.861102:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763276.861105:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763276.861108:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930000 x1874599832608640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763276.861118:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930000 x1874599832608640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.861126:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763276.861128:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.861130:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763276.861161:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763276.861167:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930000 x1874599832608640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.861176:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763276.861181:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930000 x1874599832608640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.861189:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763276.861192:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.861194:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763276.861196:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.861198:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763276.861200:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763276.861203:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763276.861208:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763276.861226:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763276.861231:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930000 x1874599832608640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763276.861241:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832608640:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763276.861246:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.861249:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763276.861252:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930000 x1874599832608640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.861260:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763276.861262:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763276.861265:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763276.861269:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763276.861271:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763276.861273:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787763276.861275:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763276.861278:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca800.
02000000:00000001:0.0:1787763276.861282:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763276.861283:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763276.861285:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930000.
00000100:00000001:0.0:1787763276.861288:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763276.861290:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.861292:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.861298:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763276.861300:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763276.861302:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763276.861304:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763276.861306:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763276.861311:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933480 x1874599832608768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.861318:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763276.861320:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763276.861322:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763276.861324:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763276.861327:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763276.861330:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832608768:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763276.861334:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763276.861336:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763276.861337:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763276.861340:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763276.861343:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763276.861345:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763276.861349:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763276.861352:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763276.861356:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763276.861360:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832608768, portal 10
00000100:00000001:0.0:1787763276.861362:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763276.861363:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497734784 : -115097211816832 : ffff9751d7933480)
00000100:00000040:0.0:1787763276.861371:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933480 x1874599832608768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763301 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.861378:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763276.861381:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763276.861383:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763276.861386:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832608768, offset 0
00000400:00000200:0.0:1787763276.861391:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763276.861395:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763276.861399:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763276.861403:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763276.861406:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763276.861409:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763276.861412:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763276.861415:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57991:57991:256:4294967295] 192.168.203.110@tcp LPNI seq info [57991:57991:8:4294967295]
00000400:00000200:0.0:1787763276.861420:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763276.861427:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763276.861432:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763276.861436:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763276.861440:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763276.861444:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787763276.861449:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763276.861452:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1787763276.861456:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763276.861458:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763276.861461:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763276.861465:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763276.861468:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763276.861470:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763276.861473:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763276.861477:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933480 x1874599832608768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763301 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.861486:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763276.861489:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933480 x1874599832608768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763301 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.861497:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763276.861499:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763276.861503:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763276.861518:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:0.0:1787763276.861522:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763276.861524:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315663904
00000100:00000001:0.0:1787763276.861527:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315665139
00000100:00000001:0.0:1787763276.861529:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315663904
00000100:00000001:0.0:1787763276.861532:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315665139
00000100:00000001:0.0:1787763276.861534:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763276.861537:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763276.861539:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763276.861544:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763276/real 1787763276]  req@ffff9751d7933480 x1874599832608768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763276.861553:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763276.861555:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763276.861557:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763276.861560:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763276.861562:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763276.861564:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763276.861568:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933480 x1874599832608768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763276.861589:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933480 x1874599832608768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.861597:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763276.861599:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.861601:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763276.861602:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763276.861606:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933480 x1874599832608768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.861614:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763276.861618:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933480 x1874599832608768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.861626:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763276.861629:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.861631:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763276.861632:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.861634:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763276.861646:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763276.861650:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763276.861653:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763276.861656:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763276.861660:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933480 x1874599832608768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763276.861669:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832608768:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763276.861673:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763276.861677:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930b40 x1874599832608896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.861684:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763276.861686:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763276.861688:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763276.861690:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763276.861692:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763276.861696:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832608896:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763276.861699:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763276.861701:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763276.861703:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763276.861705:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763276.861709:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763276.861711:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763276.861715:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763276.861718:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763276.861721:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763276.861725:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832608896, portal 10
00000100:00000001:0.0:1787763276.861727:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763276.861728:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497724224 : -115097211827392 : ffff9751d7930b40)
00000100:00000040:0.0:1787763276.861735:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930b40 x1874599832608896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763301 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.861749:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763276.861752:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763276.861755:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763276.861758:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832608896, offset 0
00000400:00000200:0.0:1787763276.861763:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763276.861767:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763276.861783:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763276.861786:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763276.861789:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763276.861792:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763276.861794:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763276.861797:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57992:57992:256:4294967295] 192.168.203.110@tcp LPNI seq info [57992:57992:8:4294967295]
00000400:00000200:0.0:1787763276.861802:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763276.861809:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763276.861814:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763276.861818:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763276.861822:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763276.861826:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787763276.861830:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763276.861833:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1787763276.861837:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763276.861840:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763276.861843:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763276.861856:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763276.861859:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763276.861861:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763276.861864:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763276.861867:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930b40 x1874599832608896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763301 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.861876:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763276.861879:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930b40 x1874599832608896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763301 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.861894:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763276.861896:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763276.861900:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763276.861903:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:0.0:1787763276.861906:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763276.861909:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315663892
00000100:00000001:0.0:1787763276.861912:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315665139
00000100:00000001:0.0:1787763276.861913:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315663892
00000100:00000001:0.0:1787763276.861916:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315665139
00000100:00000001:0.0:1787763276.861918:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763276.861920:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763276.861922:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763276.861927:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763276/real 1787763276]  req@ffff9751d7930b40 x1874599832608896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763276.861935:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763276.861937:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763276.861939:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763276.861942:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763276.861944:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763276.861946:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763276.861950:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930b40 x1874599832608896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763276.861960:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930b40 x1874599832608896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.861967:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763276.861969:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.861971:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763276.861973:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763276.861977:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930b40 x1874599832608896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.861996:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763276.862001:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930b40 x1874599832608896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.862009:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763276.862011:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.862013:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763276.862015:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.862017:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763276.862019:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763276.862022:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763276.862025:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763276.862027:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763276.862032:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930b40 x1874599832608896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763276.862041:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832608896:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763276.862046:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763276.862049:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7931680 x1874599832609024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.862057:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763276.862058:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763276.862060:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763276.862062:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763276.862064:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763276.862068:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832609024:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763276.862071:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763276.862073:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763276.862075:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763276.862077:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763276.862081:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016000.
02000000:00000001:0.0:1787763276.862083:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763276.862087:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763276.862090:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763276.862094:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763276.862097:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832609024, portal 4
00000100:00000001:0.0:1787763276.862099:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763276.862101:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497727104 : -115097211824512 : ffff9751d7931680)
00000100:00000040:0.0:1787763276.862107:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7931680 x1874599832609024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763301 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.862114:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763276.862117:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763276.862120:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763276.862122:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832609024, offset 0
00000400:00000200:0.0:1787763276.862127:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763276.862131:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763276.862165:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763276.862168:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763276.862171:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763276.862174:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763276.862177:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763276.862180:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57993:57993:256:4294967295] 192.168.203.110@tcp LPNI seq info [57993:57993:8:4294967295]
00000400:00000200:0.0:1787763276.862185:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763276.862192:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763276.862198:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763276.862202:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763276.862206:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763276.862224:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:0.1:1787763276.862230:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763276.862233:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:0.0:1787763276.862237:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763276.862240:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763276.862243:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763276.862247:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763276.862250:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763276.862252:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763276.862255:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763276.862259:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7931680 x1874599832609024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763301 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.862268:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763276.862271:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7931680 x1874599832609024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763301 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.862279:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763276.862281:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763276.862285:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763276.862288:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:0.0:1787763276.862292:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763276.862294:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315663910
00000100:00000001:0.0:1787763276.862297:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315665139
00000100:00000001:0.0:1787763276.862299:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315663910
00000100:00000001:0.0:1787763276.862301:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315665139
00000100:00000001:0.0:1787763276.862303:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763276.862306:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763276.862308:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763276.862312:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763276/real 1787763276]  req@ffff9751d7931680 x1874599832609024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763276.862322:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763276.862323:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763276.862326:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763276.862329:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763276.862331:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763276.862333:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763276.862337:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7931680 x1874599832609024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763276.862348:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7931680 x1874599832609024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.862355:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763276.862356:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.862358:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763276.862360:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763276.862364:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7931680 x1874599832609024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.862372:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763276.862376:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7931680 x1874599832609024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.862383:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763276.862385:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.862387:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763276.862389:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.862391:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763276.862392:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763276.862395:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763276.862399:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763276.862401:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763276.862405:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7931680 x1874599832609024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763276.862414:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832609024:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763276.862418:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.862421:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763276.862425:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933480 x1874599832608768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.862433:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763276.862435:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763276.862437:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763276.862440:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763276.862442:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763276.862443:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763276.862446:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763276.862448:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8c00.
02000000:00000001:0.0:1787763276.862451:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763276.862452:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763276.862454:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933480.
00000100:00000001:0.0:1787763276.862458:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763276.862460:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.862461:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763276.862465:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930b40 x1874599832608896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.862472:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763276.862474:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763276.862476:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763276.862479:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763276.862480:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763276.862482:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763276.862484:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763276.862486:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb800.
02000000:00000001:0.0:1787763276.862488:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763276.862489:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763276.862491:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930b40.
00000100:00000001:0.0:1787763276.862493:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763276.862504:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.862506:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763276.862509:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7931680 x1874599832609024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763301 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763276.862516:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763276.862518:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763276.862520:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016000.
02000000:00000001:0.0:1787763276.862522:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763276.862523:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763276.862525:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787763276.862527:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763276.862528:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca400.
02000000:00000001:0.0:1787763276.862530:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763276.862531:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763276.862533:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7931680.
00000100:00000001:0.0:1787763276.862535:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763276.862537:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.862539:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763276.862543:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763276.862545:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763276.862547:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763276.862548:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763277.500328:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763277.500333:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763277.500336:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763277.884253:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763277.884262:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763277.884265:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763277.884271:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763277.884277:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763277.884281:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763277.884285:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763277.884297:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763277.884301:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763278.908220:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763278.908226:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763278.908228:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763278.908234:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763278.908237:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763278.908240:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763278.908242:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763278.908245:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763278.908247:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763279.548393:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763279.548397:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763279.548400:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763279.932301:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763279.932313:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763279.932316:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763279.932323:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763279.932328:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763279.932333:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763279.932336:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763279.932339:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763279.932343:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763280.572430:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763280.572439:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763280.572446:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763280.956277:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763280.956286:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763280.956289:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763280.956297:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763280.956302:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763280.956307:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763280.956310:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763280.956313:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763280.956317:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763281.980743:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763281.980751:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763281.980755:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763281.980766:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bf480 x1874599832609152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.980783:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763281.980787:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763281.980793:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763281.980798:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763281.980804:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763281.980812:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832609152:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763281.980820:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763281.980827:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763281.980832:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763281.980837:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763281.980846:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763281.980851:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763281.980860:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763281.980867:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763281.980878:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763281.980888:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832609152, portal 25
00000100:00000001:0.0:1787763281.980893:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763281.980896:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049859200 : -115096659692416 : ffff9751f87bf480)
00000100:00000040:0.0:1787763281.980908:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bf480 x1874599832609152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763306 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.980926:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763281.980933:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763281.980938:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763281.980944:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832609152, offset 0
00000400:00000200:0.0:1787763281.980956:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763281.980968:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763281.980978:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763281.980987:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763281.980995:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763281.981002:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763281.981008:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763281.981016:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57994:57994:256:4294967295] 192.168.203.110@tcp LPNI seq info [57994:57994:8:4294967295]
00000400:00000200:0.0:1787763281.981026:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763281.981044:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763281.981055:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763281.981064:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763281.981076:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763281.981084:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787763281.981095:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763281.981102:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1787763281.981112:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763281.981117:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763281.981128:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763281.981205:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763281.981215:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763281.981220:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763281.981228:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763281.981236:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bf480 x1874599832609152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763306 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.981256:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763281.981263:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bf480 x1874599832609152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763306 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.981278:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763281.981282:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763281.981294:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763281.981301:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763281.981306:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315670260
00000100:00000010:0.0:1787763281.981315:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940ca80.
00000100:00000001:0.0:1787763281.981326:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763281.981336:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315670260
00000100:00000010:0.0:1787763281.981343:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c360.
00000100:00000001:0.0:1787763281.981348:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787763281.981354:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763281.981360:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763281.981365:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763281.981374:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763281/real 1787763281]  req@ffff9751f87bf480 x1874599832609152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763281.981394:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763281.981397:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763281.981401:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763281.981409:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763281.981412:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763281.981417:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763281.981425:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bf480 x1874599832609152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763281.981444:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bf480 x1874599832609152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.981459:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763281.981462:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763281.981467:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763281.981470:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763281.981479:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bf480 x1874599832609152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.981495:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763281.981505:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bf480 x1874599832609152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.981523:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763281.981528:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763281.981532:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763281.981534:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763281.981539:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763281.981543:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763281.981549:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763281.981556:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763281.981562:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763281.981570:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bf480 x1874599832609152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763281.981589:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832609152:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763281.981598:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763281.981603:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763281.981608:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bf480 x1874599832609152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.981626:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763281.981629:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763281.981633:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763281.981638:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763281.981640:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763281.981642:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787763281.981645:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763281.981648:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611c00.
02000000:00000001:0.0:1787763281.981653:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763281.981655:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763281.981657:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bf480.
00000100:00000001:0.0:1787763281.981661:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763281.981663:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763281.981666:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763281.981674:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763281.981677:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763281.981680:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763281.981682:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763281.981684:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763281.981689:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc000 x1874599832609280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.981698:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763281.981700:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763281.981702:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763281.981705:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763281.981707:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763281.981711:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832609280:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763281.981715:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763281.981718:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763281.981720:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763281.981723:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763281.981727:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763281.981730:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763281.981735:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763281.981739:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763281.981744:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763281.981748:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832609280, portal 10
00000100:00000001:0.0:1787763281.981751:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763281.981753:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049845760 : -115096659705856 : ffff9751f87bc000)
00000100:00000040:0.0:1787763281.981762:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc000 x1874599832609280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763306 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.981770:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763281.981773:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763281.981776:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763281.981780:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832609280, offset 0
00000400:00000200:0.0:1787763281.981785:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763281.981790:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763281.981795:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763281.981799:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763281.981803:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763281.981806:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763281.981809:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763281.981814:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57995:57995:256:4294967295] 192.168.203.110@tcp LPNI seq info [57995:57995:8:4294967295]
00000400:00000200:0.0:1787763281.981819:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763281.981828:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763281.981834:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763281.981838:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763281.981843:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763281.981847:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787763281.981853:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763281.981856:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1787763281.981862:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763281.981864:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763281.981868:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763281.981873:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763281.981877:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763281.981879:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763281.981883:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763281.981888:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc000 x1874599832609280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763306 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.981898:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763281.981902:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc000 x1874599832609280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763306 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.981910:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763281.981912:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763281.981917:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763281.981922:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:0.0:1787763281.981925:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763281.981928:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315669024
00000100:00000001:0.0:1787763281.981931:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315670260
00000100:00000001:0.0:1787763281.981934:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315669024
00000100:00000001:0.0:1787763281.981937:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315670260
00000100:00000001:0.0:1787763281.981940:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763281.981943:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763281.981945:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763281.981949:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763281/real 1787763281]  req@ffff9751f87bc000 x1874599832609280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763281.981959:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763281.981960:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763281.981962:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763281.981966:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763281.981967:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763281.981970:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763281.981973:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc000 x1874599832609280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763281.981984:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc000 x1874599832609280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.982000:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763281.982002:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763281.982004:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763281.982006:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763281.982010:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc000 x1874599832609280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.982018:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763281.982023:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc000 x1874599832609280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.982031:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763281.982034:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763281.982036:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763281.982037:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763281.982040:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763281.982051:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763281.982054:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763281.982058:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763281.982060:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763281.982065:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc000 x1874599832609280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763281.982075:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832609280:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763281.982080:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763281.982084:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87be1c0 x1874599832609408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.982092:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763281.982093:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763281.982096:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763281.982099:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763281.982101:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763281.982105:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832609408:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763281.982109:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763281.982111:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763281.982113:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763281.982115:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763281.982119:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763281.982121:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763281.982125:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763281.982128:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763281.982131:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763281.982167:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832609408, portal 10
00000100:00000001:0.0:1787763281.982170:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763281.982171:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049854400 : -115096659697216 : ffff9751f87be1c0)
00000100:00000040:0.0:1787763281.982206:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87be1c0 x1874599832609408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763306 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.982215:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763281.982217:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763281.982222:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763281.982225:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832609408, offset 0
00000400:00000200:0.0:1787763281.982230:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763281.982235:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763281.982239:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763281.982243:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763281.982247:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763281.982250:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763281.982253:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763281.982257:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57996:57996:256:4294967295] 192.168.203.110@tcp LPNI seq info [57996:57996:8:4294967295]
00000400:00000200:0.0:1787763281.982263:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763281.982271:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763281.982277:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763281.982281:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763281.982286:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763281.982290:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787763281.982295:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763281.982299:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1787763281.982304:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763281.982307:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763281.982311:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763281.982315:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763281.982319:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763281.982321:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763281.982324:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763281.982329:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87be1c0 x1874599832609408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763306 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.982339:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763281.982342:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87be1c0 x1874599832609408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763306 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.982351:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763281.982353:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763281.982357:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763281.982361:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:0.0:1787763281.982365:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763281.982367:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315669013
00000100:00000001:0.0:1787763281.982371:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315670260
00000100:00000001:0.0:1787763281.982373:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315669013
00000100:00000001:0.0:1787763281.982376:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315670260
00000100:00000001:0.0:1787763281.982378:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763281.982381:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763281.982383:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763281.982388:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763281/real 1787763281]  req@ffff9751f87be1c0 x1874599832609408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763281.982399:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763281.982400:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763281.982403:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763281.982406:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763281.982408:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763281.982410:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763281.982414:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87be1c0 x1874599832609408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763281.982424:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87be1c0 x1874599832609408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.982433:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763281.982434:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763281.982436:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763281.982438:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763281.982442:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87be1c0 x1874599832609408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.982450:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763281.982455:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87be1c0 x1874599832609408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.982463:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763281.982466:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763281.982467:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763281.982469:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763281.982471:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763281.982473:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763281.982476:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763281.982480:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763281.982482:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763281.982488:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87be1c0 x1874599832609408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763281.982498:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832609408:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763281.982503:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763281.982507:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bde00 x1874599832609536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.982515:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763281.982516:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763281.982519:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763281.982521:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763281.982523:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763281.982527:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832609536:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763281.982530:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763281.982533:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763281.982535:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763281.982537:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763281.982541:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9017800.
02000000:00000001:0.0:1787763281.982543:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763281.982547:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763281.982550:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763281.982553:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763281.982557:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832609536, portal 4
00000100:00000001:0.0:1787763281.982559:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763281.982561:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049853440 : -115096659698176 : ffff9751f87bde00)
00000100:00000040:0.0:1787763281.982568:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bde00 x1874599832609536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763306 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.982576:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763281.982579:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763281.982582:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763281.982585:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832609536, offset 0
00000400:00000200:0.0:1787763281.982590:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763281.982594:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763281.982598:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763281.982601:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763281.982605:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763281.982608:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763281.982611:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763281.982614:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [57997:57997:256:4294967295] 192.168.203.110@tcp LPNI seq info [57997:57997:8:4294967295]
00000400:00000200:0.0:1787763281.982620:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763281.982627:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763281.982632:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763281.982635:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763281.982640:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763281.982643:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:0.1:1787763281.982648:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763281.982653:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:0.0:1787763281.982657:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763281.982660:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763281.982663:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763281.982668:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763281.982671:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763281.982673:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763281.982676:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763281.982680:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bde00 x1874599832609536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763306 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.982689:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763281.982693:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bde00 x1874599832609536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763306 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.982701:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763281.982703:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763281.982706:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763281.982710:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:0.0:1787763281.982713:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763281.982716:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315669030
00000100:00000001:0.0:1787763281.982719:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315670260
00000100:00000001:0.0:1787763281.982722:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315669030
00000100:00000001:0.0:1787763281.982725:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315670260
00000100:00000001:0.0:1787763281.982727:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763281.982729:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763281.982731:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763281.982735:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763281/real 1787763281]  req@ffff9751f87bde00 x1874599832609536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763281.982744:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763281.982746:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763281.982747:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763281.982750:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763281.982752:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763281.982755:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763281.982759:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bde00 x1874599832609536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763281.982769:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bde00 x1874599832609536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.982776:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763281.982778:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763281.982780:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763281.982782:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763281.982786:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bde00 x1874599832609536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.982794:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763281.982798:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bde00 x1874599832609536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.982807:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763281.982810:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763281.982812:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763281.982814:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763281.982816:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763281.982818:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763281.982822:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763281.982825:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763281.982828:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763281.982832:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bde00 x1874599832609536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763281.982841:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832609536:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763281.982846:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763281.982848:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763281.982852:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc000 x1874599832609280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.982859:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763281.982861:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763281.982863:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763281.982866:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763281.982868:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763281.982870:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763281.982872:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763281.982874:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610800.
02000000:00000001:0.0:1787763281.982878:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763281.982879:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763281.982881:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc000.
00000100:00000001:0.0:1787763281.982884:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763281.982886:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763281.982888:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763281.982892:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87be1c0 x1874599832609408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.982908:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763281.982909:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763281.982911:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763281.982914:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763281.982915:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763281.982917:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763281.982919:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763281.982920:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610c00.
02000000:00000001:0.0:1787763281.982923:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763281.982924:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763281.982925:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87be1c0.
00000100:00000001:0.0:1787763281.982928:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763281.982930:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763281.982931:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763281.982935:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bde00 x1874599832609536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763306 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763281.982942:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763281.982944:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763281.982945:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9017800.
02000000:00000001:0.0:1787763281.982949:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763281.982950:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763281.982951:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787763281.982953:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763281.982955:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612000.
02000000:00000001:0.0:1787763281.982957:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763281.982959:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763281.982960:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bde00.
00000100:00000001:0.0:1787763281.982963:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763281.982964:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763281.982966:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763281.982970:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763281.982972:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763281.982974:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763281.982976:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763282.620405:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763282.620412:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763282.620419:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763283.644565:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763283.644572:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763283.644577:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763285.692439:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763285.692443:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763285.692447:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763286.716451:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763286.716458:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763286.716463:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763287.100256:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763287.100264:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763287.100267:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763287.100271:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763287.100275:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763287.100282:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21001
00000400:00000200:0.0:1787763287.100290:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763287.100296:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763287.100300:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763287.100304:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763287.100307:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763287.100309:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763287.100312:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763287.100316:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763287.100318:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763287.100321:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763287.100325:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763287.100327:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763287.100330:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763287.100333:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763287.100336:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763287.100339:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763287.100342:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763287.100345:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763287.100349:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763287.100352:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763287.100355:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763287.100358:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763287.100361:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763287.100364:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763287.100366:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763287.100375:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
00000020:00000040:0.0:1787763287.100379:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763287.100383:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763287.100385:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763287.100388:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763287.100390:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763287.100395:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763287.100403:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763287.100410:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763287.100420:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832609664/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763287.100431:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763287.100436:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763287.100441:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832609664/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763287.100463:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763287.100467:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763287.100470:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763287.100472:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763287.100474:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763287.100477:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763287.100481:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21001
00000400:00000200:0.0:1787763287.100486:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763287.100490:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763287.100493:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763287.100497:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763287.100502:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763287.100507:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763287.100511:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763287.100515:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763287.100518:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763287.100522:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763287.100526:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763287.100540:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763287.100545:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763287.100553:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763287.100559:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763287.100563:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763287.100569:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763287.100573:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763287.100578:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763287.100584:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763287.100589:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763287.100595:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763287.100601:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763287.100606:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763287.100613:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763287.100618:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763287.100623:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763287.100629:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787763287.100636:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763287.100643:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763287.100647:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763287.100651:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763287.100654:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763287.100657:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763287.100664:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787763287.100671:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763287.100682:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832609792/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763287.100697:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763287.100701:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763287.100707:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832609792/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763287.100721:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763287.100726:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763287.100732:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763287.100735:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763287.100737:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763287.100739:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763287.100745:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21001
00000400:00000200:0.0:1787763287.100752:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763287.100760:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763287.100766:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763287.100774:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763287.100779:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763287.100785:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763287.100790:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763287.100793:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763287.100795:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763287.100799:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763287.100804:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763287.100808:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763287.100814:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763287.100822:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763287.100828:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763287.100830:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763287.100833:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763287.100839:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763287.100843:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763287.100849:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763287.100853:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763287.100858:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763287.100864:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763287.100868:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:3.1:1787763287.100870:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000008:00100000:0.0:1787763287.100871:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000800:00000200:3.1:1787763287.100878:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000008:00000001:0.0:1787763287.100879:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763287.100883:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763287.100889:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000020:00000040:0.0:1787763287.100895:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763287.100900:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763287.100902:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763287.100905:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763287.100909:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763287.100912:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763287.100918:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787763287.100924:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763287.100933:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832609920/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763287.100948:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763287.100952:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763287.100962:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832609920/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763287.100976:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763287.100982:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763287.100992:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763287.100997:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763287.101003:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763287.101007:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763287.101009:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763287.101012:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763287.101017:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21001
00000400:00000200:0.0:1787763287.101025:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763287.101031:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763287.101036:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763287.101040:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763287.101044:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763287.101047:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763287.101050:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763287.101055:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763287.101057:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763287.101059:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763287.101064:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763287.101068:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763287.101073:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763287.101080:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763287.101085:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763287.101088:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763287.101093:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763287.101099:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763287.101105:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763287.101110:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763287.101115:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763287.101119:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763287.101124:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763287.101127:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763287.101190:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763287.101197:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763287.101201:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763287.101228:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763287.101235:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763287.101241:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763287.101244:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763287.101248:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763287.101252:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763287.101255:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763287.101262:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763287.101267:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763287.101278:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832610048/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763287.101296:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763287.101302:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763287.101310:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832610048/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763287.101327:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763287.101334:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21011)
00000800:00000200:3.1:1787763287.101559:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763287.101561:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787763287.101895:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763287.101898:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787763287.102245:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763287.102248:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000020:00000001:0.0:1787763288.764300:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763288.764304:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763288.764306:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763289.788386:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763289.788394:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763289.788400:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763291.836451:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763291.836458:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763291.836463:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763292.220331:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763292.220342:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763292.220348:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763292.220354:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763292.220361:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763292.220371:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21006
00000400:00000200:0.0:1787763292.220382:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763292.220393:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763292.220398:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763292.220403:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763292.220408:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763292.220412:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763292.220416:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763292.220421:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763292.220424:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763292.220428:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763292.220432:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763292.220436:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763292.220439:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763292.220443:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763292.220445:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763292.220448:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763292.220450:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763292.220454:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763292.220458:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763292.220461:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763292.220464:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763292.220468:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763292.220472:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763292.220475:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763292.220478:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763292.220486:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763292.220492:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763292.220497:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763292.220499:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763292.220503:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763292.220506:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763292.220511:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763292.220520:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763292.220528:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763292.220541:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832610176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763292.220554:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763292.220560:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763292.220566:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832610176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763292.220593:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763292.220597:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763292.220602:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763292.220604:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763292.220606:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763292.220611:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763292.220615:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21006
00000400:00000200:0.0:1787763292.220620:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763292.220625:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763292.220630:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763292.220637:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763292.220643:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763292.220651:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763292.220657:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763292.220663:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763292.220667:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763292.220673:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763292.220679:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763292.220688:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763292.220697:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763292.220709:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763292.220716:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763292.220721:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763292.220741:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763292.220749:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763292.220760:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763292.220773:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763292.220780:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763292.220789:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763292.220801:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763292.220804:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763292.220812:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763292.220819:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763292.220825:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763292.220832:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763292.220840:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763292.220853:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763292.220861:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763292.220871:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763292.220878:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763292.220887:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763292.220905:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787763292.220919:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763292.220941:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832610304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763292.220981:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763292.220994:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763292.221017:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832610304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763292.221079:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763292.221086:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:3.1:1787763292.221097:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00080000:0.0:1787763292.221099:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763292.221131:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763292.221164:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763292.221169:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763292.221176:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21006
00000400:00000200:0.0:1787763292.221190:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763292.221203:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763292.221209:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763292.221218:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763292.221223:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763292.221230:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763292.221234:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763292.221243:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763292.221245:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763292.221249:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763292.221253:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763292.221262:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763292.221266:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763292.221277:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763292.221284:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763292.221290:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763292.221297:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763292.221305:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763292.221315:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763292.221322:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763292.221326:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763292.221335:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763292.221344:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763292.221352:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763292.221361:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763292.221373:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763292.221381:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763292.221395:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763292.221406:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763292.221417:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763292.221422:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763292.221431:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763292.221435:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763292.221444:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763292.221456:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763292.221467:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763292.221483:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832610432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763292.221509:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763292.221516:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763292.221531:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832610432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763292.221570:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763292.221582:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763292.221595:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763292.221607:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763292.221618:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763292.221626:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763292.221634:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763292.221643:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763292.221655:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21006
00000400:00000200:0.0:1787763292.221668:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763292.221686:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763292.221697:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763292.221711:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763292.221722:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763292.221736:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763292.221747:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763292.221759:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763292.221766:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763292.221776:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763292.221782:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763292.221791:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763292.221799:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000800:00000200:3.1:1787763292.221812:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763292.221815:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763292.221826:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763292.221836:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763292.221848:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763292.221859:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763292.221875:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763292.221887:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763292.221896:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763292.221907:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763292.221921:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763292.221931:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763292.221942:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763292.221954:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763292.221963:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763292.221976:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763292.221988:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763292.222001:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763292.222008:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763292.222017:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763292.222024:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763292.222030:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763292.222047:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763292.222062:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763292.222084:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832610560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763292.222180:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763292.222186:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763292.222197:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832610560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763292.222234:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763292.222249:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21016)
00000800:00000200:3.1:1787763292.222443:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763292.222983:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787763292.860401:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763292.860408:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763292.860413:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763294.908382:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763294.908388:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763294.908393:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763295.293024:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763295.293037:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763295.293048:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763295.293077:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763292/real 1787763295]  req@ffff9751f876c000 x1874599832610304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763317 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763295.293196:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763295.293201:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763295.293207:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc234b00.
00000400:00000200:0.0:1787763295.293217:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32c38
00000400:00000010:0.0:1787763295.293221:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32c38.
00000100:00000001:0.0:1787763295.293241:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763295.293252:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832610304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763295.293285:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832610304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763295.293297:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763295.293301:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763295.293331:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763295.293334:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763295.293342:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832610304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763295.293355:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763295.293364:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832610304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763295.293379:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763295.293385:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763295.293389:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763295.293391:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763295.293396:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763295.293422:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763295.293430:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763295.293440:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763295.293445:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763295.293454:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832610304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763295.293471:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832610304:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763295.293481:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763295.293491:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763292/real 1787763295]  req@ffff9751f876e1c0 x1874599832610176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763295.293545:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763295.293551:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763295.293555:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc234528.
00000400:00000200:0.0:1787763295.293561:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32440
00000400:00000010:0.0:1787763295.293565:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32440.
00000100:00000001:0.0:1787763295.293570:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763295.293579:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832610176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763295.293597:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832610176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763295.293610:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763295.293613:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763295.293617:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763295.293622:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763295.293631:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832610176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763295.293645:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763295.293653:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832610176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763295.293666:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763295.293671:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763295.293675:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763295.293679:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763295.293683:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763295.293687:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763295.293693:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763295.293701:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763295.293706:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763295.293715:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832610176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763295.293731:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832610176:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763295.293739:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763295.293748:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763295.293763:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832610304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763295.293790:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763295.293800:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763295.293811:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613c00.
02000000:00000001:0.0:1787763295.293823:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763295.293848:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763295.293856:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763295.293866:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763295.293877:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014400.
02000000:00000001:0.0:1787763295.293896:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763295.293903:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763295.293910:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:0.0:1787763295.293919:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763295.293925:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763295.293936:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763295.293949:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832610176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763295.294015:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763295.294023:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763295.294033:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610400.
02000000:00000001:0.0:1787763295.294045:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763295.294052:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763295.294061:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787763295.294069:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763295.294073:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014800.
02000000:00000001:0.0:1787763295.294079:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763295.294107:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763295.294117:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:0.0:1787763295.294127:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763295.294130:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763295.294181:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763295.294186:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763295.294191:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763295.294195:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763295.294209:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763292/real 1787763295]  req@ffff9751f876ed00 x1874599832610560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763295.294229:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763295.294231:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763295.294234:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc234c08.
00000400:00000200:0.0:1787763295.294240:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32dd0
00000400:00000010:0.0:1787763295.294244:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32dd0.
00000100:00000001:0.0:1787763295.294250:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763295.294256:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832610560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763295.294269:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832610560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763295.294281:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763295.294283:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763295.294286:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763295.294289:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763295.294295:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832610560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763295.294307:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763295.294314:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832610560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763295.294326:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763295.294331:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763295.294333:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763295.294335:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763295.294339:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763295.294342:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763295.294347:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763295.294353:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763295.294358:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763295.294365:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832610560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763295.294379:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832610560:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763295.294387:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763295.294396:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763292/real 1787763295]  req@ffff9751f876e580 x1874599832610432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763295.294410:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763295.294413:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763295.294416:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc234688.
00000400:00000200:0.0:1787763295.294421:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32198
00000400:00000010:0.0:1787763295.294425:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32198.
00000100:00000001:0.0:1787763295.294429:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763295.294436:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832610432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763295.294452:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832610432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763295.294465:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763295.294468:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763295.294472:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763295.294475:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763295.294482:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832610432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763295.294490:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763295.294496:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832610432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763295.294504:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763295.294507:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763295.294509:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763295.294511:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763295.294513:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763295.294515:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763295.294518:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763295.294521:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763295.294524:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763295.294528:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832610432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763295.294538:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832610432:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763295.294545:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763295.294549:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763295.294555:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832610560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763295.294568:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763295.294571:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763295.294574:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611800.
02000000:00000001:0.0:1787763295.294597:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763295.294600:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763295.294603:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787763295.294606:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763295.294610:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016800.
02000000:00000001:0.0:1787763295.294616:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763295.294618:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763295.294621:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:0.0:1787763295.294627:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763295.294630:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763295.294634:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763295.294641:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832610432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763317 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763295.294654:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763295.294657:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763295.294660:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613800.
02000000:00000001:0.0:1787763295.294666:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763295.294668:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763295.294671:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763295.294675:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763295.294678:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015c00.
02000000:00000001:0.0:1787763295.294682:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763295.294685:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763295.294687:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:0.0:1787763295.294707:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763295.294709:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763295.294711:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763295.294720:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763295.294723:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763295.294725:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763295.294727:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763295.932364:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763295.932371:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763295.932376:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763297.340291:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763297.340305:0:259241:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763297.340309:0:259241:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763297.340316:0:259241:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763297.340324:0:259241:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763297.340340:0:259241:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21011
00000400:00000200:0.0:1787763297.340363:0:259241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763297.340383:0:259241:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763297.340393:0:259241:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763297.340402:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763297.340410:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763297.340414:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763297.340419:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763297.340426:0:259241:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763297.340430:0:259241:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763297.340436:0:259241:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763297.340442:0:259241:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763297.340446:0:259241:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763297.340451:0:259241:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763297.340457:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763297.340461:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763297.340465:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763297.340468:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763297.340473:0:259241:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763297.340479:0:259241:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763297.340485:0:259241:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763297.340490:0:259241:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763297.340497:0:259241:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763297.340503:0:259241:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763297.340507:0:259241:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763297.340512:0:259241:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763297.340522:0:259241:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763297.340529:0:259241:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763297.340536:0:259241:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763297.340539:0:259241:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763297.340544:0:259241:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763297.340549:0:259241:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763297.340555:0:259241:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763297.340567:0:259241:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763297.340577:0:259241:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763297.340593:0:259241:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832610688/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763297.340624:0:259241:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763297.340631:0:259241:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763297.340641:0:259241:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832610688/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763297.340680:0:259241:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763297.340687:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763297.340693:0:259241:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763297.340696:0:259241:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763297.340700:0:259241:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763297.340706:0:259241:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763297.340711:0:259241:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21011
00000400:00000200:0.0:1787763297.340719:0:259241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763297.340726:0:259241:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763297.340732:0:259241:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763297.340738:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763297.340744:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763297.340750:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763297.340755:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763297.340760:0:259241:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763297.340764:0:259241:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763297.340775:0:259241:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763297.340780:0:259241:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763297.340790:0:259241:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763297.340798:0:259241:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763297.340810:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763297.340822:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763297.340832:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763297.340842:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763297.340856:0:259241:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763297.340873:0:259241:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763297.340884:0:259241:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763297.340893:0:259241:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763297.340907:0:259241:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763297.340921:0:259241:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763297.340932:0:259241:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763297.340944:0:259241:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763297.340959:0:259241:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763297.340967:0:259241:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763297.340978:0:259241:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763297.340991:0:259241:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763297.341004:0:259241:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763297.341012:0:259241:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763297.341019:0:259241:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763297.341028:0:259241:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763297.341033:0:259241:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763297.341045:0:259241:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763297.341057:0:259241:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763297.341098:0:259241:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832610816/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763297.341128:0:259241:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763297.341241:0:259241:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763297.341263:0:259241:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832610816/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763297.341292:0:259241:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763297.341304:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763297.341318:0:259241:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763297.341325:0:259241:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763297.341332:0:259241:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763297.341345:0:259241:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763297.341361:0:259241:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21011
00000400:00000200:0.0:1787763297.341379:0:259241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763297.341393:0:259241:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763297.341414:0:259241:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763297.341430:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763297.341441:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763297.341446:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763297.341451:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763297.341457:0:259241:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763297.341459:0:259241:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763297.341462:0:259241:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763297.341467:0:259241:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763297.341471:0:259241:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763297.341476:0:259241:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763297.341482:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763297.341486:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763297.341490:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763297.341494:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763297.341499:0:259241:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763297.341504:0:259241:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763297.341510:0:259241:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763297.341513:0:259241:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763297.341520:0:259241:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763297.341525:0:259241:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763297.341529:0:259241:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763297.341533:0:259241:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763297.341538:0:259241:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763297.341541:0:259241:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763297.341550:0:259241:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763297.341555:0:259241:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763297.341560:0:259241:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763297.341564:0:259241:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763297.341568:0:259241:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763297.341571:0:259241:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763297.341575:0:259241:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763297.341583:0:259241:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763297.341590:0:259241:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763297.341600:0:259241:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832610944/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763297.341615:0:259241:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763297.341621:0:259241:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763297.341630:0:259241:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832610944/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763297.341648:0:259241:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763297.341659:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763297.341673:0:259241:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763297.341687:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763297.341701:0:259241:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000800:00000200:3.1:1787763297.341709:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763297.341712:0:259241:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763297.341719:0:259241:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:3.1:1787763297.341723:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:0.0:1787763297.341731:0:259241:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763297.341748:0:259241:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21011
00000400:00000200:0.0:1787763297.341765:0:259241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763297.341780:0:259241:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763297.341802:0:259241:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763297.341820:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763297.341832:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763297.341843:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763297.341856:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763297.341869:0:259241:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763297.341876:0:259241:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763297.341885:0:259241:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763297.341894:0:259241:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763297.341903:0:259241:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763297.341916:0:259241:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763297.341934:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763297.341943:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763297.341956:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763297.341962:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763297.341967:0:259241:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763297.341989:0:259241:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763297.342001:0:259241:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763297.342013:0:259241:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763297.342022:0:259241:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763297.342032:0:259241:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763297.342037:0:259241:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763297.342047:0:259241:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763297.342057:0:259241:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763297.342091:0:259241:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763297.342103:0:259241:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763297.342115:0:259241:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763297.342123:0:259241:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763297.342131:0:259241:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763297.342181:0:259241:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763297.342187:0:259241:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763297.342191:0:259241:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763297.342202:0:259241:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787763297.342214:0:259241:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763297.342231:0:259241:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832611072/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763297.342255:0:259241:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763297.342266:0:259241:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763297.342289:0:259241:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832611072/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763297.342345:0:259241:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763297.342360:0:259241:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21022)
00000800:00000200:3.1:1787763297.342736:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763297.342742:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787763297.343281:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763297.343284:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787763297.343645:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763297.343650:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:0.0:1787763297.980408:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763297.980415:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763297.980420:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763299.004475:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763299.004481:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763299.004486:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763301.052379:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763301.052386:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763301.052391:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763302.076361:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763302.076366:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763302.076369:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763302.460286:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763302.460294:0:259241:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763302.460296:0:259241:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763302.460301:0:259241:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763302.460306:0:259241:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763302.460317:0:259241:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21017
00000400:00000200:0.0:1787763302.460329:0:259241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763302.460336:0:259241:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763302.460341:0:259241:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763302.460345:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763302.460350:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763302.460355:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763302.460359:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763302.460363:0:259241:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763302.460365:0:259241:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763302.460368:0:259241:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763302.460374:0:259241:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763302.460376:0:259241:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763302.460378:0:259241:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763302.460381:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763302.460383:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763302.460385:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763302.460386:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763302.460390:0:259241:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763302.460396:0:259241:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763302.460399:0:259241:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763302.460403:0:259241:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763302.460408:0:259241:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763302.460412:0:259241:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763302.460416:0:259241:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763302.460420:0:259241:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763302.460431:0:259241:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000020:00000040:0.0:1787763302.460435:0:259241:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763302.460440:0:259241:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763302.460444:0:259241:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763302.460450:0:259241:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763302.460454:0:259241:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763302.460459:0:259241:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763302.460468:0:259241:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763302.460478:0:259241:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763302.460494:0:259241:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832611200/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763302.460507:0:259241:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763302.460514:0:259241:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763302.460520:0:259241:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832611200/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763302.460546:0:259241:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763302.460549:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763302.460552:0:259241:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763302.460553:0:259241:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763302.460555:0:259241:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763302.460559:0:259241:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763302.460563:0:259241:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21017
00000400:00000200:0.0:1787763302.460568:0:259241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763302.460570:0:259241:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763302.460573:0:259241:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763302.460577:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763302.460580:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763302.460587:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763302.460594:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763302.460596:0:259241:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763302.460601:0:259241:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763302.460607:0:259241:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763302.460611:0:259241:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763302.460615:0:259241:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763302.460620:0:259241:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763302.460625:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763302.460628:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763302.460636:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763302.460640:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763302.460645:0:259241:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763302.460652:0:259241:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763302.460660:0:259241:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763302.460664:0:259241:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763302.460673:0:259241:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763302.460680:0:259241:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763302.460685:0:259241:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763302.460690:0:259241:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763302.460696:0:259241:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763302.460700:0:259241:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763302.460706:0:259241:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000020:00000040:0.0:1787763302.460710:0:259241:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763302.460715:0:259241:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763302.460717:0:259241:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763302.460720:0:259241:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763302.460723:0:259241:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763302.460725:0:259241:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763302.460730:0:259241:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787763302.460734:0:259241:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763302.460738:0:259241:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832611328/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763302.460745:0:259241:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763302.460748:0:259241:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763302.460752:0:259241:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832611328/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763302.460760:0:259241:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763302.460764:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763302.460770:0:259241:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763302.460772:0:259241:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763302.460775:0:259241:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763302.460779:0:259241:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763302.460783:0:259241:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21017
00000400:00000200:0.0:1787763302.460789:0:259241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763302.460793:0:259241:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763302.460798:0:259241:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763302.460804:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763302.460806:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763302.460809:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763302.460816:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763302.460820:0:259241:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763302.460823:0:259241:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763302.460825:0:259241:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763302.460828:0:259241:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763302.460829:0:259241:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763302.460831:0:259241:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763302.460834:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763302.460836:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763302.460837:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763302.460838:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763302.460840:0:259241:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763302.460841:0:259241:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763302.460843:0:259241:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763302.460844:0:259241:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763302.460846:0:259241:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763302.460848:0:259241:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763302.460849:0:259241:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763302.460850:0:259241:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763302.460852:0:259241:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763302.460853:0:259241:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763302.460857:0:259241:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
00000020:00000040:0.0:1787763302.460858:0:259241:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763302.460860:0:259241:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763302.460861:0:259241:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763302.460863:0:259241:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763302.460864:0:259241:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763302.460865:0:259241:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763302.460868:0:259241:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787763302.460871:0:259241:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763302.460875:0:259241:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832611456/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763302.460886:0:259241:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763302.460887:0:259241:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763302.460890:0:259241:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832611456/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763302.460896:0:259241:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763302.460898:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763302.460901:0:259241:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763302.460903:0:259241:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763302.460906:0:259241:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763302.460907:0:259241:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763302.460908:0:259241:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763302.460910:0:259241:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763302.460912:0:259241:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21017
00000400:00000200:0.0:1787763302.460914:0:259241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763302.460916:0:259241:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763302.460918:0:259241:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763302.460919:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763302.460921:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763302.460922:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763302.460923:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763302.460925:0:259241:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763302.460926:0:259241:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763302.460927:0:259241:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763302.460929:0:259241:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763302.460931:0:259241:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763302.460932:0:259241:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763302.460935:0:259241:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763302.460937:0:259241:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763302.460938:0:259241:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763302.460940:0:259241:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763302.460942:0:259241:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763302.460944:0:259241:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763302.460946:0:259241:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763302.460947:0:259241:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763302.460948:0:259241:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763302.460949:0:259241:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763302.460951:0:259241:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763302.460953:0:259241:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763302.460954:0:259241:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763302.460955:0:259241:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763302.460959:0:259241:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763302.460960:0:259241:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763302.460962:0:259241:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763302.460963:0:259241:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763302.460964:0:259241:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763302.460965:0:259241:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763302.460966:0:259241:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763302.460968:0:259241:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763302.460973:0:259241:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763302.460980:0:259241:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832611584/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763302.460988:0:259241:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763302.460992:0:259241:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763302.460998:0:259241:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832611584/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763302.461011:0:259241:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763302.461015:0:259241:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21027)
00000800:00000200:3.1:1787763302.461465:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763302.461468:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787763302.461789:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763302.461791:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787763302.461990:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763302.461992:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787763302.462231:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763302.462233:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:0.0:1787763303.484288:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763303.484298:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763303.484302:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763303.484309:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763303.484314:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763303.484318:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763303.484322:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763303.484325:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763303.484329:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763304.124454:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763304.124459:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763304.124463:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763304.508279:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763304.508293:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763304.508298:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763304.508309:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763304.508316:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763304.508321:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763304.508325:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763304.508327:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763304.508333:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763305.148434:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763305.148442:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763305.148449:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763305.532271:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763305.532283:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763305.532288:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763305.532297:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763305.532303:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763305.532309:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763305.532314:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763305.532318:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763305.532323:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763306.556293:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763306.556306:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763306.556310:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763306.556319:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763306.556325:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763306.556332:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763306.556337:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763306.556342:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763306.556347:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763307.196349:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763307.196356:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763307.196362:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763307.580283:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763307.580296:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763307.580303:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763307.580310:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763307.580319:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763307.580332:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21022
00000400:00000200:0.0:1787763307.580355:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763307.580367:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763307.580371:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763307.580376:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763307.580382:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763307.580386:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763307.580389:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763307.580407:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763307.580410:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763307.580414:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763307.580418:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763307.580422:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763307.580425:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763307.580429:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763307.580431:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763307.580434:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763307.580436:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763307.580440:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763307.580444:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763307.580447:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763307.580452:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763307.580457:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763307.580461:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763307.580464:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763307.580467:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763307.580479:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763307.580483:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763307.580489:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763307.580491:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763307.580495:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763307.580498:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763307.580503:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763307.580513:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787763307.580521:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763307.580532:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832611712/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763307.580546:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763307.580551:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763307.580557:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832611712/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763307.580587:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763307.580591:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763307.580596:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763307.580598:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763307.580600:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763307.580604:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763307.580607:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21022
00000400:00000200:0.0:1787763307.580612:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763307.580615:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763307.580618:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763307.580622:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763307.580625:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763307.580628:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763307.580631:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763307.580634:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763307.580637:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763307.580639:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763307.580643:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763307.580646:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763307.580649:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763307.580655:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763307.580659:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763307.580662:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763307.580664:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763307.580669:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763307.580677:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763307.580682:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763307.580692:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763307.580708:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763307.580718:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763307.580724:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763307.580732:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763307.580749:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763307.580762:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763307.580786:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763307.580798:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763307.580805:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763307.580808:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763307.580816:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763307.580825:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763307.580831:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763307.580845:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787763307.580862:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763307.580881:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832611840/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763307.580918:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763307.580927:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763307.580968:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832611840/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763307.581030:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763307.581048:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763307.581060:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763307.581068:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763307.581075:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763307.581081:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763307.581094:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21022
00000400:00000200:0.0:1787763307.581116:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763307.581126:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763307.581181:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763307.581188:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763307.581201:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763307.581218:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763307.581228:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763307.581240:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763307.581251:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763307.581260:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763307.581267:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763307.581272:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763307.581290:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763307.581303:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763307.581310:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763307.581329:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763307.581342:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:3.1:1787763307.581342:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:0.0:1787763307.581360:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000800:00000200:3.1:1787763307.581360:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000100:00080000:0.0:1787763307.581379:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763307.581392:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763307.581400:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763307.581406:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763307.581417:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763307.581421:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763307.581425:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763307.581431:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763307.581438:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763307.581446:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763307.581450:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763307.581456:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763307.581460:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763307.581465:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763307.581467:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763307.581470:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763307.581486:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763307.581493:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763307.581510:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832611968/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763307.581525:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763307.581529:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763307.581538:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832611968/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763307.581612:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763307.581619:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763307.581626:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763307.581632:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763307.581640:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763307.581645:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763307.581652:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763307.581657:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763307.581668:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21022
00000400:00000200:0.0:1787763307.581674:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763307.581687:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763307.581692:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763307.581705:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763307.581711:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763307.581719:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763307.581725:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763307.581732:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763307.581740:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763307.581743:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763307.581748:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763307.581761:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763307.581776:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763307.581795:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763307.581811:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763307.581819:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763307.581829:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763307.581847:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763307.581870:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763307.581877:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763307.581883:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763307.581890:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763307.581906:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763307.581927:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763307.581942:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763307.581960:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763307.581981:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763307.582019:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763307.582034:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763307.582049:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763307.582057:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763307.582068:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763307.582079:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763307.582086:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763307.582106:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787763307.582119:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763307.582167:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832612096/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763307.582215:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763307.582222:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763307.582237:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832612096/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763307.582288:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763307.582300:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21032)
00000800:00000200:3.1:1787763307.582456:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763307.582457:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787763307.582667:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763307.582669:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787763307.582854:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763307.582856:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:0.0:1787763308.220409:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763308.220416:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763308.220421:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763309.628322:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763309.628334:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763309.628338:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763309.628347:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763309.628353:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763309.628359:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763309.628365:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763309.628369:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763309.628375:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763310.268454:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763310.268459:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763310.268463:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763310.652224:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763310.652235:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763310.652238:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763310.652245:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763310.652249:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763310.652253:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763310.652256:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763310.652259:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763310.652262:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763311.292429:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763311.292434:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763311.292437:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763311.676252:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763311.676262:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763311.676265:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763311.676272:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763311.676279:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763311.676285:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763311.676289:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763311.676293:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763311.676297:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763312.700271:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763312.700287:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763312.700291:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763312.700298:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763312.700304:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763312.700314:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763312.700317:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763312.700322:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763312.700326:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763312.700939:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763312.700945:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787763312.701450:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763312.701453:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787763312.701674:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763312.701676:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787763312.701868:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763312.701870:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:0.0:1787763313.340446:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763313.340455:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763313.340461:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763313.724227:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763313.724236:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763313.724238:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763313.724246:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763313.724250:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763313.724254:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763313.724258:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763313.724262:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763313.724266:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763314.364344:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763314.364348:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763314.364352:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763314.748263:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763314.748276:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763314.748279:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763314.748287:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763314.748292:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763314.748298:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763314.748301:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763314.748305:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763314.748309:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763315.772223:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763315.772230:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763315.772232:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763315.772237:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763315.772242:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763315.772246:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763315.772249:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763315.772252:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763315.772254:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763316.412309:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763316.412313:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763316.412316:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763316.796251:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763316.796259:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763316.796262:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763316.796269:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763316.796277:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763316.796282:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763316.796285:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763316.796288:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763316.796293:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763317.436437:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763317.436444:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763317.436449:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763317.820275:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763317.820288:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763317.820292:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763317.820299:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763317.820312:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763317.820327:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21032
00000400:00000200:0.0:1787763317.820344:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763317.820357:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763317.820365:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763317.820370:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763317.820374:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763317.820377:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763317.820381:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763317.820390:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763317.820394:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763317.820400:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763317.820405:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763317.820414:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763317.820418:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763317.820423:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763317.820426:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763317.820428:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763317.820431:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763317.820435:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763317.820441:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763317.820444:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763317.820448:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763317.820453:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763317.820457:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763317.820459:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763317.820464:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763317.820476:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000020:00000040:0.0:1787763317.820481:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763317.820486:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763317.820488:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763317.820509:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763317.820512:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763317.820528:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763317.820538:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787763317.820544:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763317.820554:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832612736/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763317.820565:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763317.820570:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763317.820575:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832612736/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763317.820597:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763317.820601:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763317.820604:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763317.820606:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763317.820608:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763317.820610:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763317.820613:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21032
00000400:00000200:0.0:1787763317.820617:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763317.820620:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763317.820623:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763317.820625:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763317.820628:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763317.820630:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763317.820632:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763317.820635:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763317.820638:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763317.820640:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763317.820643:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763317.820646:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763317.820648:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763317.820652:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763317.820654:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763317.820657:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763317.820663:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763317.820668:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763317.820675:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763317.820679:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763317.820683:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763317.820690:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763317.820700:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763317.820707:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763317.820714:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763317.820725:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763317.820730:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763317.820737:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787763317.820743:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763317.820749:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763317.820754:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763317.820758:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763317.820762:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763317.820765:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763317.820773:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763317.820781:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763317.820791:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832612864/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763317.820813:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763317.820818:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763317.820824:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832612864/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763317.820837:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763317.820842:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763317.820853:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763317.820858:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763317.820862:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763317.820871:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763317.820880:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21032
00000400:00000200:0.0:1787763317.820887:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763317.820896:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763317.820903:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763317.820911:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763317.820915:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763317.820923:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763317.820926:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763317.820948:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763317.820951:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763317.820955:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763317.820962:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763317.820965:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763317.820988:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763317.821004:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763317.821011:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763317.821023:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763317.821031:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763317.821040:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763317.821049:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763317.821059:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763317.821068:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763317.821075:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763317.821080:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763317.821089:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763317.821097:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763317.821108:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763317.821113:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763317.821125:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
00000020:00000040:0.0:1787763317.821133:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763317.821144:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763317.821146:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763317.821183:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763317.821186:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763317.821188:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763317.821195:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787763317.821218:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763317.821225:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832612992/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763317.821240:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763317.821247:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763317.821254:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832612992/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763317.821270:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763317.821271:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763317.821274:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763317.821282:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:3.1:1787763317.821283:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000100:00080000:0.0:1787763317.821286:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763317.821293:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763317.821314:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763317.821319:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763317.821322:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763317.821328:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21032
00000400:00000200:0.0:1787763317.821338:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763317.821346:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763317.821351:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763317.821362:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763317.821373:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763317.821380:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763317.821388:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763317.821395:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763317.821398:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763317.821401:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763317.821406:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763317.821413:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763317.821422:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763317.821437:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763317.821448:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763317.821455:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763317.821462:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763317.821469:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763317.821483:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763317.821493:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763317.821503:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763317.821524:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763317.821529:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763317.821533:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763317.821537:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763317.821540:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763317.821543:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763317.821553:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763317.821558:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763317.821566:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763317.821569:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763317.821576:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763317.821578:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763317.821580:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763317.821586:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763317.821592:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763317.821601:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832613120/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763317.821614:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763317.821618:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763317.821630:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832613120/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763317.821654:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763317.821662:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21042)
00000800:00000200:3.1:1787763317.822049:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763317.822053:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787763317.822584:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763317.822588:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787763317.823097:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763317.823101:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000020:00000001:0.0:1787763319.484344:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763319.484349:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763319.484353:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763320.508376:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763320.508383:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763320.508389:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763322.556339:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763322.556342:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763322.556344:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763322.940262:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763322.940272:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763322.940276:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763322.940282:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763322.940288:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763322.940302:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21037
00000400:00000200:0.0:1787763322.940319:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763322.940327:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763322.940331:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763322.940338:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763322.940343:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763322.940346:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763322.940348:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763322.940352:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763322.940354:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763322.940357:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763322.940361:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763322.940363:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763322.940364:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763322.940366:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763322.940368:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763322.940370:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763322.940371:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763322.940374:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763322.940377:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763322.940379:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763322.940382:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763322.940385:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763322.940388:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763322.940390:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763322.940392:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763322.940404:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763322.940406:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763322.940410:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763322.940412:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763322.940415:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763322.940418:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763322.940423:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763322.940432:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763322.940440:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763322.940450:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832613248/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763322.940464:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763322.940469:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763322.940473:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832613248/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763322.940496:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763322.940499:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763322.940501:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763322.940503:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763322.940504:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763322.940506:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763322.940509:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21037
00000400:00000200:0.0:1787763322.940511:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763322.940513:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763322.940515:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763322.940518:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763322.940519:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763322.940522:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763322.940523:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763322.940526:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763322.940527:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763322.940528:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763322.940530:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763322.940531:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763322.940533:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763322.940535:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763322.940536:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763322.940537:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763322.940539:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763322.940542:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763322.940545:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763322.940547:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763322.940551:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763322.940557:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763322.940560:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763322.940563:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763322.940569:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763322.940576:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763322.940580:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763322.940584:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763322.940589:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763322.940592:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763322.940594:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763322.940598:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763322.940599:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763322.940601:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763322.940608:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763322.940612:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763322.940621:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832613376/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763322.940631:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763322.940634:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763322.940637:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832613376/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763322.940645:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763322.940649:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763322.940652:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763322.940653:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763322.940655:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763322.940661:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763322.940666:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21037
00000400:00000200:0.0:1787763322.940673:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763322.940679:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763322.940681:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763322.940687:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763322.940688:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763322.940689:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763322.940690:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763322.940693:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763322.940695:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763322.940698:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763322.940699:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763322.940701:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763322.940702:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763322.940706:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763322.940709:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763322.940713:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763322.940717:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763322.940719:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763322.940724:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763322.940730:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763322.940735:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763322.940739:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763322.940741:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763322.940744:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763322.940748:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763322.940752:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763322.940757:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763322.940763:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763322.940766:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763322.940770:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763322.940773:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763322.940775:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763322.940776:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763322.940777:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763322.940782:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763322.940785:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763322.940791:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832613504/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763322.940798:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763322.940802:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763322.940810:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832613504/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763322.940821:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763322.940824:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763322.940829:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763322.940835:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763322.940843:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763322.940847:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763322.940852:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763322.940854:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763322.940857:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21037
00000400:00000200:0.0:1787763322.940868:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763322.940874:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763322.940877:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763322.940883:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763322.940886:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763322.940887:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763322.940889:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763322.940891:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763322.940911:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763322.940913:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763322.940918:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763322.940923:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763322.940927:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763322.940931:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763322.940936:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763322.940938:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763322.940941:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:3.1:1787763322.940943:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:0.0:1787763322.940944:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763322.940950:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000800:00000200:3.1:1787763322.940952:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000100:00000001:0.0:1787763322.940953:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763322.940958:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763322.940962:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763322.940965:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763322.940968:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763322.940973:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763322.940976:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763322.940977:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763322.940981:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763322.940986:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763322.940991:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763322.940992:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763322.940994:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763322.940998:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763322.941001:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763322.941006:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787763322.941011:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763322.941018:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832613632/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763322.941029:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763322.941033:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763322.941040:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832613632/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763322.941055:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763322.941062:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21047)
00000800:00000200:3.1:1787763322.941360:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763322.941362:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787763322.941668:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763322.941670:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787763322.941865:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763322.941867:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000020:00000001:0.0:1787763323.580465:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763323.580470:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763323.580473:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763323.964281:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763323.964291:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763323.964293:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763323.964300:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763323.964304:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763323.964307:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763323.964310:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763323.964313:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763323.964315:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763324.988275:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763324.988285:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763324.988288:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763324.988296:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763324.988302:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763324.988307:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763324.988310:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763324.988314:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763324.988319:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763325.628426:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763325.628433:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763325.628438:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763326.012271:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763326.012285:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763326.012290:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763326.012300:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763326.012309:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763326.012314:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763326.012319:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763326.012323:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763326.012327:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763326.652454:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763326.652461:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763326.652466:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763327.036293:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763327.036304:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763327.036307:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763327.036314:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763327.036320:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763327.036324:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763327.036328:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763327.036333:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763327.036337:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763328.060260:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763328.060270:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763328.060275:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763328.060281:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763328.060288:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763328.060294:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.060301:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763328.060304:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.060312:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763328.060568:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763328.060576:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763328.060580:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763328.060592:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bed00 x1874599832613760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.060617:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763328.060622:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763328.060633:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763328.060641:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763328.060650:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763328.060660:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832613760:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763328.060672:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763328.060683:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763328.060690:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763328.060697:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763328.060708:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9017000.
02000000:00000001:0.0:1787763328.060713:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763328.060727:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763328.060735:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763328.060749:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763328.060763:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832613760, portal 25
00000100:00000001:0.0:1787763328.060768:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763328.060772:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049857280 : -115096659694336 : ffff9751f87bed00)
00000100:00000040:0.0:1787763328.060788:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bed00 x1874599832613760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763353 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.060810:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763328.060818:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763328.060822:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763328.060831:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832613760, offset 0
00000400:00000200:0.0:1787763328.060846:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763328.060878:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763328.060894:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763328.060905:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763328.060912:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763328.060922:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763328.060930:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763328.060940:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58030:58030:256:4294967295] 192.168.203.110@tcp LPNI seq info [58030:58030:8:4294967295]
00000400:00000200:0.0:1787763328.060948:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763328.060968:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763328.060984:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763328.060995:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763328.061009:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763328.061015:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787763328.061027:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763328.061038:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:0.0:1787763328.061046:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763328.061050:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763328.061059:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763328.061069:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763328.061076:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763328.061080:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763328.061088:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763328.061096:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bed00 x1874599832613760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763353 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.061114:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763328.061121:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bed00 x1874599832613760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763353 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.061160:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763328.061162:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763328.061173:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763328.061178:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763328.061182:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315716340
00000100:00000010:0.0:1787763328.061192:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c7e0.
00000100:00000001:0.0:1787763328.061204:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763328.061217:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315716340
00000100:00000010:0.0:1787763328.061224:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c4e0.
00000100:00000001:0.0:1787763328.061234:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787763328.061241:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763328.061250:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763328.061256:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763328.061265:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763328/real 1787763328]  req@ffff9751f87bed00 x1874599832613760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763328.061284:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763328.061289:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763328.061293:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763328.061303:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763328.061306:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763328.061314:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763328.061320:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bed00 x1874599832613760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763328.061348:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bed00 x1874599832613760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.061375:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763328.061382:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.061389:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763328.061395:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763328.061408:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bed00 x1874599832613760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.061422:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763328.061431:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bed00 x1874599832613760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.061440:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763328.061444:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.061446:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763328.061448:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.061450:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763328.061453:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763328.061455:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763328.061460:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763328.061462:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763328.061467:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bed00 x1874599832613760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763328.061474:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832613760:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763328.061478:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.061480:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763328.061482:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bed00 x1874599832613760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.061494:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763328.061496:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763328.061499:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9017000.
02000000:00000001:0.0:1787763328.061504:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763328.061506:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763328.061508:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787763328.061510:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763328.061513:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612c00.
02000000:00000001:0.0:1787763328.061517:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763328.061519:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763328.061521:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bed00.
00000100:00000001:0.0:1787763328.061524:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763328.061525:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.061527:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.061535:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763328.061536:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763328.061538:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763328.061540:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763328.061542:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763328.061545:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bfc00 x1874599832613888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.061551:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763328.061552:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763328.061554:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763328.061555:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763328.061557:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763328.061559:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832613888:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763328.061562:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763328.061564:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763328.061566:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763328.061567:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763328.061570:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9017000.
02000000:00000001:0.0:1787763328.061572:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763328.061576:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763328.061579:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763328.061582:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763328.061585:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832613888, portal 10
00000100:00000001:0.0:1787763328.061587:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763328.061589:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049861120 : -115096659690496 : ffff9751f87bfc00)
00000100:00000040:0.0:1787763328.061595:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bfc00 x1874599832613888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763353 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.061601:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763328.061603:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763328.061605:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763328.061607:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832613888, offset 0
00000400:00000200:0.0:1787763328.061611:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763328.061614:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763328.061618:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763328.061621:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763328.061624:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763328.061627:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763328.061630:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763328.061634:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58031:58031:256:4294967295] 192.168.203.110@tcp LPNI seq info [58031:58031:8:4294967295]
00000400:00000200:0.0:1787763328.061640:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763328.061650:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763328.061656:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763328.061661:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763328.061666:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763328.061670:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787763328.061677:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763328.061681:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:0.0:1787763328.061685:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763328.061688:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763328.061692:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763328.061696:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763328.061700:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763328.061701:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763328.061705:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763328.061708:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bfc00 x1874599832613888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763353 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.061716:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763328.061718:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bfc00 x1874599832613888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763353 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.061724:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763328.061726:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763328.061729:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763328.061733:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:0.0:1787763328.061736:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763328.061738:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315715104
00000100:00000001:0.0:1787763328.061741:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315716340
00000100:00000001:0.0:1787763328.061743:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315715104
00000100:00000001:0.0:1787763328.061745:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315716340
00000100:00000001:0.0:1787763328.061747:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763328.061749:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763328.061750:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763328.061753:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763328/real 1787763328]  req@ffff9751f87bfc00 x1874599832613888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763328.061759:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763328.061760:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763328.061761:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763328.061764:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763328.061766:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763328.061767:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763328.061770:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bfc00 x1874599832613888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763328.061777:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bfc00 x1874599832613888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.061788:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763328.061790:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.061791:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763328.061792:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763328.061795:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bfc00 x1874599832613888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.061800:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763328.061803:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bfc00 x1874599832613888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.061808:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763328.061810:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.061812:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763328.061813:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.061814:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763328.061828:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763328.061830:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763328.061833:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763328.061835:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763328.061838:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bfc00 x1874599832613888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763328.061844:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832613888:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763328.061847:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763328.061850:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc3c0 x1874599832614016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.061855:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763328.061856:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763328.061873:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763328.061875:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763328.061877:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763328.061880:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832614016:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763328.061884:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763328.061886:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763328.061887:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763328.061889:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763328.061892:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763328.061894:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763328.061897:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763328.061900:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763328.061903:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763328.061906:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832614016, portal 10
00000100:00000001:0.0:1787763328.061908:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763328.061909:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049846720 : -115096659704896 : ffff9751f87bc3c0)
00000100:00000040:0.0:1787763328.061915:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc3c0 x1874599832614016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763353 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.061920:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763328.061922:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763328.061924:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763328.061926:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832614016, offset 0
00000400:00000200:0.0:1787763328.061929:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763328.061932:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763328.061935:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763328.061938:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763328.061941:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763328.061943:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763328.061945:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763328.061948:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58032:58032:256:4294967295] 192.168.203.110@tcp LPNI seq info [58032:58032:8:4294967295]
00000400:00000200:0.0:1787763328.061951:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763328.061956:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763328.061961:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763328.061964:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763328.061967:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763328.061970:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787763328.061973:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763328.061975:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:0.0:1787763328.061979:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763328.061981:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763328.061983:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763328.061987:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763328.061989:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763328.061991:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763328.061993:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763328.061996:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc3c0 x1874599832614016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763353 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.062003:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763328.062005:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc3c0 x1874599832614016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763353 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.062011:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763328.062012:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763328.062015:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763328.062018:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:0.0:1787763328.062020:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763328.062022:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315715092
00000100:00000001:0.0:1787763328.062024:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315716340
00000100:00000001:0.0:1787763328.062026:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315715092
00000100:00000001:0.0:1787763328.062028:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315716340
00000100:00000001:0.0:1787763328.062031:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763328.062032:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763328.062034:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763328.062037:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763328/real 1787763328]  req@ffff9751f87bc3c0 x1874599832614016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763328.062044:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763328.062045:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763328.062047:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763328.062049:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763328.062050:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763328.062052:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763328.062054:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc3c0 x1874599832614016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763328.062061:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc3c0 x1874599832614016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.062067:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763328.062068:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.062069:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763328.062070:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763328.062073:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc3c0 x1874599832614016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.062078:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763328.062081:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc3c0 x1874599832614016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.062086:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763328.062088:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.062089:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763328.062090:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.062091:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763328.062092:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763328.062094:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763328.062097:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763328.062099:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763328.062103:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc3c0 x1874599832614016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763328.062110:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832614016:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763328.062113:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763328.062115:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc780 x1874599832614144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.062120:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763328.062121:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763328.062123:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763328.062124:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763328.062126:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763328.062128:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832614144:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763328.062130:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763328.062131:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763328.062152:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763328.062154:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763328.062157:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763328.062159:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763328.062162:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763328.062164:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763328.062167:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763328.062169:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832614144, portal 4
00000100:00000001:0.0:1787763328.062171:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763328.062172:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049847680 : -115096659703936 : ffff9751f87bc780)
00000100:00000040:0.0:1787763328.062178:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc780 x1874599832614144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763353 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.062183:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763328.062185:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763328.062187:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763328.062189:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832614144, offset 0
00000400:00000200:0.0:1787763328.062193:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763328.062196:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763328.062198:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763328.062201:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763328.062203:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763328.062205:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763328.062207:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763328.062210:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58033:58033:256:4294967295] 192.168.203.110@tcp LPNI seq info [58033:58033:8:4294967295]
00000400:00000200:0.0:1787763328.062214:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763328.062219:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763328.062223:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763328.062225:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763328.062228:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763328.062230:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:0.1:1787763328.062234:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763328.062236:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:0.0:1787763328.062239:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763328.062241:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763328.062244:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763328.062246:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763328.062249:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763328.062251:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763328.062252:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763328.062255:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc780 x1874599832614144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763353 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.062262:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763328.062265:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc780 x1874599832614144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763353 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.062270:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763328.062272:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763328.062275:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763328.062277:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:0.0:1787763328.062279:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763328.062281:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315715110
00000100:00000001:0.0:1787763328.062284:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315716340
00000100:00000001:0.0:1787763328.062286:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315715110
00000100:00000001:0.0:1787763328.062288:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315716340
00000100:00000001:0.0:1787763328.062289:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763328.062291:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763328.062292:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763328.062295:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763328/real 1787763328]  req@ffff9751f87bc780 x1874599832614144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763328.062301:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763328.062302:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763328.062304:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763328.062306:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763328.062307:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763328.062309:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763328.062312:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc780 x1874599832614144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763328.062319:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc780 x1874599832614144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.062323:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763328.062324:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.062326:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763328.062327:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763328.062329:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc780 x1874599832614144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.062334:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763328.062337:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc780 x1874599832614144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.062342:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763328.062344:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.062346:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763328.062347:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.062349:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763328.062350:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763328.062352:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763328.062355:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763328.062356:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763328.062359:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc780 x1874599832614144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763328.062365:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832614144:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763328.062368:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.062371:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763328.062375:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bfc00 x1874599832613888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.062384:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763328.062386:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763328.062388:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9017000.
02000000:00000001:0.0:1787763328.062392:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763328.062394:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763328.062396:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763328.062398:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763328.062401:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613000.
02000000:00000001:0.0:1787763328.062404:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763328.062406:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763328.062409:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bfc00.
00000100:00000001:0.0:1787763328.062412:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763328.062414:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.062416:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763328.062421:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc3c0 x1874599832614016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.062439:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763328.062441:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763328.062442:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763328.062446:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763328.062447:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763328.062448:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763328.062450:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763328.062451:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612400.
02000000:00000001:0.0:1787763328.062453:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763328.062454:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763328.062455:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc3c0.
00000100:00000001:0.0:1787763328.062457:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763328.062458:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.062459:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763328.062462:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc780 x1874599832614144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763353 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763328.062467:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763328.062468:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763328.062469:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763328.062471:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763328.062472:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763328.062473:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787763328.062475:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763328.062476:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612800.
02000000:00000001:0.0:1787763328.062477:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763328.062478:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763328.062479:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc780.
00000100:00000001:0.0:1787763328.062481:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763328.062482:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.062483:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763328.062486:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763328.062487:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763328.062489:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763328.062490:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763328.700379:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763328.700384:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763328.700388:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763329.084322:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763329.084335:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763329.084341:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763329.084350:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763329.084357:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763329.084361:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763329.084365:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763329.084368:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763329.084372:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763329.724521:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763329.724529:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763329.724534:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763330.108217:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763330.108226:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763330.108228:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763330.108234:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763330.108238:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763330.108241:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763330.108244:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763330.108248:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763330.108250:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763331.132324:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763331.132338:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763331.132342:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763331.132353:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763331.132362:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763331.132370:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763331.132376:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763331.132381:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763331.132388:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763331.772357:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763331.772362:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763331.772366:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763332.156335:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763332.156347:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763332.156351:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763332.156361:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763332.156368:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763332.156374:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763332.156379:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763332.156384:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763332.156389:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763332.796350:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763332.796355:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763332.796359:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763333.180356:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763333.180369:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763333.180379:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763333.180392:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763333.180405:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763333.180415:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.180426:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763333.180434:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.180446:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763333.181610:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763333.181620:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763333.181629:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763333.181660:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832614272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.181684:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763333.181691:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763333.181697:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763333.181703:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763333.181709:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763333.181716:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832614272:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763333.181722:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763333.181727:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763333.181730:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763333.181734:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763333.181744:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763333.181749:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763333.181759:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763333.181767:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763333.181778:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763333.181789:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832614272, portal 25
00000100:00000001:0.0:1787763333.181793:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763333.181796:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:0.0:1787763333.181808:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832614272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763358 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.181849:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763333.181855:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763333.181861:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763333.181867:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832614272, offset 0
00000400:00000200:0.0:1787763333.181880:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763333.181892:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763333.181920:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763333.181934:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763333.181941:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763333.181946:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763333.181950:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763333.181954:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58034:58034:256:4294967295] 192.168.203.110@tcp LPNI seq info [58034:58034:8:4294967295]
00000400:00000200:0.0:1787763333.181960:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763333.181972:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763333.181979:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763333.181987:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763333.181995:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763333.182000:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1787763333.182006:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763333.182011:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000100:0.0:1787763333.182016:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763333.182019:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763333.182025:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763333.182030:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763333.182036:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763333.182039:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763333.182044:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763333.182050:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832614272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763358 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.182068:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763333.182072:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832614272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763358 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.182080:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763333.182082:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763333.182092:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763333.182096:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763333.182100:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315721460
00000100:00000010:0.0:1787763333.182107:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940ccc0.
00000100:00000001:0.0:1787763333.182116:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763333.182125:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315721460
00000100:00000010:0.0:1787763333.182163:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c9c0.
00000100:00000001:0.0:1787763333.182172:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787763333.182180:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763333.182186:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763333.182202:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763333.182219:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763333/real 1787763333]  req@ffff9751f86583c0 x1874599832614272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763333.182236:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763333.182240:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763333.182243:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763333.182251:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763333.182254:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763333.182258:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763333.182265:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832614272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763333.182282:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832614272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.182305:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763333.182309:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.182313:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763333.182317:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763333.182323:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832614272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.182331:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763333.182336:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832614272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.182344:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763333.182347:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.182349:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763333.182351:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.182353:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763333.182356:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763333.182359:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763333.182363:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763333.182365:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763333.182370:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832614272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763333.182379:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832614272:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763333.182385:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763333.182389:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832614400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.182396:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763333.182398:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763333.182401:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763333.182404:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763333.182407:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763333.182410:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832614400:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763333.182414:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763333.182416:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763333.182418:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763333.182420:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763333.182424:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763333.182427:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763333.182431:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763333.182434:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763333.182438:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763333.182442:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832614400, portal 10
00000100:00000001:0.0:1787763333.182444:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763333.182446:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000040:0.0:1787763333.182453:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832614400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763358 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.182461:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763333.182463:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763333.182466:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763333.182469:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832614400, offset 0
00000400:00000200:0.0:1787763333.182473:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763333.182478:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763333.182482:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763333.182486:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763333.182489:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763333.182492:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763333.182495:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763333.182498:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58035:58035:256:4294967295] 192.168.203.110@tcp LPNI seq info [58035:58035:8:4294967295]
00000400:00000200:0.0:1787763333.182503:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763333.182511:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763333.182516:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763333.182520:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763333.182525:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763333.182528:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1787763333.182533:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763333.182536:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000100:0.0:1787763333.182540:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763333.182542:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763333.182546:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763333.182550:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763333.182553:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763333.182555:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763333.182558:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763333.182562:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832614400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763358 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.182571:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763333.182574:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832614400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763358 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.182581:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763333.182583:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763333.182587:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763333.182591:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:0.0:1787763333.182595:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763333.182597:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315720225
00000100:00000001:0.0:1787763333.182600:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315721460
00000100:00000001:0.0:1787763333.182603:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315720225
00000100:00000001:0.0:1787763333.182605:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315721460
00000100:00000001:0.0:1787763333.182607:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763333.182609:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763333.182611:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763333.182615:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763333/real 1787763333]  req@ffff9751f8658780 x1874599832614400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763333.182631:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763333.182632:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763333.182634:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763333.182637:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763333.182639:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763333.182641:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763333.182644:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832614400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763333.182653:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832614400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.182660:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763333.182661:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.182663:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763333.182664:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763333.182668:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832614400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.182691:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763333.182696:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832614400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.182703:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763333.182706:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.182708:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763333.182709:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.182711:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763333.182724:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763333.182726:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763333.182730:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763333.182732:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763333.182737:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832614400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763333.182765:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832614400:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763333.182770:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763333.182774:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832614528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.182782:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763333.182784:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763333.182786:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763333.182788:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763333.182791:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763333.182794:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832614528:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763333.182798:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763333.182801:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763333.182803:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763333.182805:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763333.182809:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9017000.
02000000:00000001:0.0:1787763333.182812:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763333.182816:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763333.182818:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763333.182864:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763333.182868:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832614528, portal 10
00000100:00000001:0.0:1787763333.182870:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763333.182872:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:0.0:1787763333.182879:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832614528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763358 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.182886:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763333.182889:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763333.182891:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763333.182894:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832614528, offset 0
00000400:00000200:0.0:1787763333.182899:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763333.182914:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763333.182918:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763333.182921:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763333.182924:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763333.182927:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763333.182929:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763333.182933:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58036:58036:256:4294967295] 192.168.203.110@tcp LPNI seq info [58036:58036:8:4294967295]
00000400:00000200:0.0:1787763333.182938:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763333.182945:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763333.182949:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763333.182970:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763333.182975:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763333.182978:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1787763333.182983:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763333.182987:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000100:0.0:1787763333.182992:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763333.182994:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763333.182998:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763333.183002:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763333.183007:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763333.183009:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763333.183012:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763333.183017:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832614528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763358 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.183026:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763333.183047:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832614528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763358 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.183054:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763333.183056:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763333.183060:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763333.183063:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:0.0:1787763333.183067:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763333.183070:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315720213
00000100:00000001:0.0:1787763333.183072:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315721460
00000100:00000001:0.0:1787763333.183075:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315720213
00000100:00000001:0.0:1787763333.183077:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315721460
00000100:00000001:0.0:1787763333.183079:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763333.183081:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763333.183083:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763333.183087:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763333/real 1787763333]  req@ffff9751f8659a40 x1874599832614528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763333.183095:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763333.183097:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763333.183098:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763333.183101:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763333.183103:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763333.183105:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763333.183109:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832614528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763333.183118:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832614528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.183124:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763333.183125:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.183127:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763333.183129:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763333.183132:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832614528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.183139:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763333.183143:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832614528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.183150:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763333.183154:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.183157:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763333.183160:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.183162:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763333.183194:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763333.183198:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763333.183202:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763333.183204:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763333.183208:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832614528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763333.183218:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832614528:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763333.183222:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.183224:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763333.183227:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832614272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.183235:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763333.183237:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763333.183240:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763333.183244:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763333.183246:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763333.183248:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787763333.183250:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763333.183253:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:0.0:1787763333.183257:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763333.183259:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763333.183262:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:0.0:1787763333.183265:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763333.183266:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.183268:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763333.183272:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832614400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.183285:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763333.183286:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763333.183288:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763333.183291:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763333.183292:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763333.183294:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763333.183296:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763333.183297:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:0.0:1787763333.183299:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763333.183300:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763333.183302:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:0.0:1787763333.183304:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763333.183305:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.183307:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763333.183310:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832614528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.183316:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763333.183317:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763333.183319:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9017000.
02000000:00000001:0.0:1787763333.183322:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763333.183323:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763333.183324:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763333.183326:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763333.183328:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:0.0:1787763333.183330:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763333.183331:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763333.183332:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:0.0:1787763333.183334:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763333.183336:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.183338:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.183345:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763333.183347:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763333.183349:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763333.183350:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763333.183352:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763333.183356:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832614656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.183363:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763333.183365:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763333.183368:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763333.183369:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763333.183372:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763333.183375:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832614656:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763333.183378:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763333.183380:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763333.183382:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763333.183384:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763333.183388:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9017000.
02000000:00000001:0.0:1787763333.183390:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763333.183393:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763333.183396:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763333.183400:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763333.183403:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832614656, portal 4
00000100:00000001:0.0:1787763333.183405:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763333.183406:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:0.0:1787763333.183412:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832614656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763358 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.183420:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763333.183422:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763333.183424:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763333.183427:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832614656, offset 0
00000400:00000200:0.0:1787763333.183431:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763333.183435:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763333.183439:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763333.183442:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763333.183445:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763333.183448:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763333.183450:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763333.183454:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58037:58037:256:4294967295] 192.168.203.110@tcp LPNI seq info [58037:58037:8:4294967295]
00000400:00000200:0.0:1787763333.183459:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763333.183466:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763333.183471:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763333.183474:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763333.183479:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763333.183482:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:0.1:1787763333.183486:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763333.183490:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000100:0.0:1787763333.183494:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763333.183496:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763333.183499:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763333.183503:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763333.183506:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763333.183508:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763333.183511:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763333.183514:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832614656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763358 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.183523:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763333.183526:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832614656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763358 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.183533:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763333.183535:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763333.183538:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763333.183541:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:0.0:1787763333.183544:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763333.183546:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315720231
00000100:00000001:0.0:1787763333.183549:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315721460
00000100:00000001:0.0:1787763333.183551:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315720231
00000100:00000001:0.0:1787763333.183553:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315721460
00000100:00000001:0.0:1787763333.183556:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763333.183558:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763333.183560:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763333.183564:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763333/real 1787763333]  req@ffff9751f8658000 x1874599832614656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763333.183573:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763333.183574:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763333.183576:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763333.183578:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763333.183580:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763333.183582:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763333.183585:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832614656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763333.183594:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832614656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.183601:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763333.183602:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.183604:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763333.183605:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763333.183609:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832614656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.183616:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763333.183620:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832614656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.183626:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763333.183629:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.183630:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763333.183632:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.183634:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763333.183635:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763333.183638:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763333.183642:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763333.183644:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763333.183649:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832614656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763333.183657:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832614656:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763333.183660:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.183662:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763333.183665:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832614656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763358 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763333.183690:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763333.183691:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763333.183693:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9017000.
02000000:00000001:0.0:1787763333.183696:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763333.183697:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763333.183699:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787763333.183701:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763333.183703:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:0.0:1787763333.183705:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763333.183706:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763333.183708:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:0.0:1787763333.183710:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763333.183712:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.183714:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763333.183717:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763333.183719:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763333.183721:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763333.183723:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763334.204243:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763334.204254:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763334.204257:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763334.204265:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763334.204270:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763334.204274:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763334.204277:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763334.204281:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763334.204286:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763334.844444:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763334.844450:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763334.844456:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763335.228287:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763335.228296:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763335.228299:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763335.228306:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763335.228311:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763335.228316:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763335.228320:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763335.228323:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763335.228327:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763335.868309:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763335.868314:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763335.868318:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763337.916481:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763337.916487:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763337.916491:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763338.300673:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763338.300686:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763338.300695:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763338.300727:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933840 x1874599832614784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.300772:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763338.300798:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763338.300807:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763338.300816:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763338.300835:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763338.300856:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832614784:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763338.300873:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763338.300880:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763338.300888:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763338.300894:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763338.300911:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763338.300927:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763338.300950:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763338.300967:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763338.300983:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763338.301006:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832614784, portal 25
00000100:00000001:0.0:1787763338.301019:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763338.301026:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497735744 : -115097211815872 : ffff9751d7933840)
00000100:00000040:0.0:1787763338.301052:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933840 x1874599832614784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763363 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.301094:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763338.301111:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763338.301129:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763338.301206:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832614784, offset 0
00000400:00000200:0.0:1787763338.301239:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763338.301269:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763338.301307:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763338.301330:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763338.301348:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763338.301367:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763338.301381:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763338.301403:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58038:58038:256:4294967295] 192.168.203.110@tcp LPNI seq info [58038:58038:8:4294967295]
00000400:00000200:0.0:1787763338.301428:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763338.301473:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763338.301505:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763338.301527:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763338.301550:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763338.301564:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787763338.301596:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763338.301612:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1787763338.301641:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763338.301653:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763338.301668:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763338.301701:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763338.301716:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763338.301728:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763338.301741:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763338.301768:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933840 x1874599832614784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763363 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.301814:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763338.301826:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933840 x1874599832614784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763363 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.301870:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763338.301888:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763338.301908:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763338.301933:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763338.301945:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315726580
00000100:00000010:0.0:1787763338.301964:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c6c0.
00000100:00000001:0.0:1787763338.301977:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763338.301988:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315726580
00000100:00000010:0.0:1787763338.301991:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940cb40.
00000100:00000001:0.0:1787763338.301995:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787763338.301998:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763338.302002:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763338.302005:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763338.302011:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763338/real 1787763338]  req@ffff9751d7933840 x1874599832614784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763338.302020:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763338.302022:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763338.302024:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763338.302029:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763338.302030:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763338.302033:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763338.302035:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933840 x1874599832614784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763338.302047:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933840 x1874599832614784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302052:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763338.302053:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763338.302055:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763338.302057:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763338.302060:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933840 x1874599832614784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302064:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763338.302068:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933840 x1874599832614784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302074:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763338.302076:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763338.302077:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763338.302078:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763338.302081:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763338.302083:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763338.302085:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763338.302089:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763338.302091:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763338.302094:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933840 x1874599832614784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763338.302099:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832614784:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763338.302103:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763338.302105:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763338.302107:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933840 x1874599832614784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302117:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763338.302120:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763338.302122:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763338.302126:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763338.302128:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763338.302130:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787763338.302132:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763338.302166:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:0.0:1787763338.302171:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763338.302174:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763338.302177:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933840.
00000100:00000001:0.0:1787763338.302183:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763338.302184:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763338.302188:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763338.302196:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763338.302198:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763338.302204:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763338.302206:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763338.302207:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763338.302211:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932580 x1874599832614912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302216:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763338.302217:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763338.302219:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763338.302222:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763338.302223:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763338.302225:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832614912:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763338.302228:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763338.302230:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763338.302231:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763338.302233:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763338.302236:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763338.302237:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763338.302240:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763338.302243:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763338.302245:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763338.302248:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832614912, portal 10
00000100:00000001:0.0:1787763338.302249:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763338.302250:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497730944 : -115097211820672 : ffff9751d7932580)
00000100:00000040:0.0:1787763338.302256:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932580 x1874599832614912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763363 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302261:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763338.302263:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763338.302264:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763338.302266:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832614912, offset 0
00000400:00000200:0.0:1787763338.302269:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763338.302272:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763338.302276:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763338.302278:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763338.302280:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763338.302282:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763338.302284:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763338.302287:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58039:58039:256:4294967295] 192.168.203.110@tcp LPNI seq info [58039:58039:8:4294967295]
00000400:00000200:0.0:1787763338.302289:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763338.302294:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763338.302297:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763338.302300:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763338.302303:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763338.302305:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787763338.302310:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763338.302312:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1787763338.302316:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763338.302318:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763338.302320:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763338.302323:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763338.302325:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763338.302326:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763338.302329:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763338.302332:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932580 x1874599832614912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763363 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302337:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763338.302339:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932580 x1874599832614912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763363 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302344:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763338.302345:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763338.302348:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763338.302351:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:0.0:1787763338.302353:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763338.302354:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315725345
00000100:00000001:0.0:1787763338.302356:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315726580
00000100:00000001:0.0:1787763338.302358:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315725345
00000100:00000001:0.0:1787763338.302360:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315726580
00000100:00000001:0.0:1787763338.302362:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763338.302363:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763338.302365:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763338.302367:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763338/real 1787763338]  req@ffff9751d7932580 x1874599832614912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763338.302371:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763338.302372:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763338.302373:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763338.302375:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763338.302376:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763338.302377:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763338.302379:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932580 x1874599832614912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763338.302385:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932580 x1874599832614912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302395:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763338.302396:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763338.302397:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763338.302398:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763338.302400:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932580 x1874599832614912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302404:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763338.302407:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932580 x1874599832614912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302411:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763338.302412:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763338.302413:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763338.302414:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763338.302415:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763338.302428:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763338.302430:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763338.302432:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763338.302433:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763338.302436:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932580 x1874599832614912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763338.302442:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832614912:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763338.302444:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763338.302447:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930780 x1874599832615040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302451:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763338.302452:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763338.302455:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763338.302456:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763338.302458:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763338.302459:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832615040:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763338.302462:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763338.302463:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763338.302464:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763338.302466:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763338.302468:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763338.302470:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763338.302472:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763338.302474:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763338.302476:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763338.302478:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832615040, portal 10
00000100:00000001:0.0:1787763338.302479:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763338.302480:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497723264 : -115097211828352 : ffff9751d7930780)
00000100:00000040:0.0:1787763338.302484:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930780 x1874599832615040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763363 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302489:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763338.302491:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763338.302492:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763338.302494:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832615040, offset 0
00000400:00000200:0.0:1787763338.302496:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763338.302498:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763338.302500:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763338.302502:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763338.302504:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763338.302505:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763338.302507:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763338.302509:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58040:58040:256:4294967295] 192.168.203.110@tcp LPNI seq info [58040:58040:8:4294967295]
00000400:00000200:0.0:1787763338.302512:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763338.302516:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763338.302520:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763338.302522:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763338.302525:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763338.302527:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787763338.302530:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763338.302532:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1787763338.302535:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763338.302536:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763338.302538:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763338.302541:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763338.302543:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763338.302544:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763338.302546:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763338.302548:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930780 x1874599832615040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763363 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302553:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763338.302555:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930780 x1874599832615040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763363 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302559:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763338.302560:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763338.302562:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763338.302564:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:0.0:1787763338.302566:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763338.302567:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315725333
00000100:00000001:0.0:1787763338.302568:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315726580
00000100:00000001:0.0:1787763338.302570:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315725333
00000100:00000001:0.0:1787763338.302572:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315726580
00000100:00000001:0.0:1787763338.302573:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763338.302575:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763338.302576:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763338.302578:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763338/real 1787763338]  req@ffff9751d7930780 x1874599832615040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763338.302585:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763338.302586:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763338.302588:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763338.302589:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763338.302590:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763338.302591:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763338.302593:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930780 x1874599832615040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763338.302599:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930780 x1874599832615040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302604:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763338.302605:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763338.302606:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763338.302607:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763338.302609:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930780 x1874599832615040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302613:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763338.302615:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930780 x1874599832615040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302619:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763338.302620:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763338.302621:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763338.302622:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763338.302623:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763338.302624:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763338.302626:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763338.302628:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763338.302629:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763338.302632:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930780 x1874599832615040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763338.302638:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832615040:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763338.302640:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763338.302643:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932940 x1874599832615168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302647:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763338.302648:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763338.302649:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763338.302650:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763338.302651:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763338.302653:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832615168:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763338.302655:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763338.302656:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763338.302657:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763338.302658:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763338.302661:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9017800.
02000000:00000001:0.0:1787763338.302662:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763338.302665:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763338.302666:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763338.302669:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763338.302671:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832615168, portal 4
00000100:00000001:0.0:1787763338.302672:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763338.302673:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497731904 : -115097211819712 : ffff9751d7932940)
00000100:00000040:0.0:1787763338.302677:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932940 x1874599832615168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763363 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302682:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763338.302683:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763338.302684:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763338.302686:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832615168, offset 0
00000400:00000200:0.0:1787763338.302689:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763338.302691:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763338.302693:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763338.302695:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763338.302697:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763338.302699:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763338.302701:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763338.302703:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58041:58041:256:4294967295] 192.168.203.110@tcp LPNI seq info [58041:58041:8:4294967295]
00000400:00000200:0.0:1787763338.302706:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763338.302709:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763338.302712:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763338.302714:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763338.302716:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763338.302718:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:0.1:1787763338.302721:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763338.302723:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:0.0:1787763338.302726:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763338.302728:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763338.302729:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763338.302732:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763338.302734:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763338.302735:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763338.302737:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763338.302739:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932940 x1874599832615168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763363 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302744:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763338.302746:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932940 x1874599832615168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763363 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302750:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763338.302751:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763338.302753:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763338.302755:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:0.0:1787763338.302757:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763338.302758:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315725350
00000100:00000001:0.0:1787763338.302760:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315726580
00000100:00000001:0.0:1787763338.302762:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315725350
00000100:00000001:0.0:1787763338.302764:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315726580
00000100:00000001:0.0:1787763338.302765:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763338.302766:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763338.302767:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763338.302769:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763338/real 1787763338]  req@ffff9751d7932940 x1874599832615168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763338.302774:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763338.302775:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763338.302776:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763338.302777:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763338.302778:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763338.302779:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763338.302781:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932940 x1874599832615168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763338.302786:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932940 x1874599832615168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302803:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763338.302804:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763338.302806:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763338.302807:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763338.302809:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932940 x1874599832615168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302814:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763338.302817:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932940 x1874599832615168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302821:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763338.302824:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763338.302825:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763338.302826:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763338.302827:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763338.302828:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763338.302831:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763338.302833:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763338.302834:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763338.302836:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932940 x1874599832615168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763338.302841:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832615168:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763338.302843:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763338.302844:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763338.302846:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932580 x1874599832614912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302850:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763338.302852:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763338.302853:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763338.302855:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763338.302856:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763338.302857:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763338.302859:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763338.302860:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb000.
02000000:00000001:0.0:1787763338.302862:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763338.302863:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763338.302864:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932580.
00000100:00000001:0.0:1787763338.302867:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763338.302868:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763338.302869:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763338.302871:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930780 x1874599832615040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302880:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763338.302881:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763338.302882:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763338.302884:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763338.302885:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763338.302885:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763338.302886:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763338.302887:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9400.
02000000:00000001:0.0:1787763338.302888:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763338.302889:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763338.302890:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930780.
00000100:00000001:0.0:1787763338.302891:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763338.302892:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763338.302893:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763338.302895:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932940 x1874599832615168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763363 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763338.302898:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763338.302899:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763338.302900:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9017800.
02000000:00000001:0.0:1787763338.302901:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763338.302902:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763338.302903:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787763338.302904:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763338.302905:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cac00.
02000000:00000001:0.0:1787763338.302906:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763338.302907:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763338.302908:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932940.
00000100:00000001:0.0:1787763338.302909:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763338.302910:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763338.302911:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763338.302912:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763338.302913:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763338.302915:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763338.302916:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763338.940398:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763338.940405:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763338.940410:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763340.988421:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763340.988429:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763340.988434:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763342.012506:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763342.012512:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763342.012517:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763343.420276:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763343.420290:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763343.420293:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763343.420298:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763343.420306:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763343.420319:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21058
00000400:00000200:0.0:1787763343.420350:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763343.420365:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763343.420371:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763343.420378:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763343.420383:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763343.420389:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763343.420393:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763343.420400:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763343.420403:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763343.420407:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763343.420412:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763343.420414:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763343.420417:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763343.420421:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763343.420424:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763343.420426:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763343.420428:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763343.420432:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763343.420437:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763343.420441:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763343.420445:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763343.420450:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763343.420453:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763343.420456:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763343.420459:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763343.420469:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
00000020:00000040:0.0:1787763343.420474:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763343.420479:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763343.420481:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763343.420485:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763343.420488:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763343.420494:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763343.420503:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763343.420512:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763343.420524:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832615296/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763343.420538:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763343.420543:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763343.420549:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832615296/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763343.420584:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763343.420588:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763343.420593:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763343.420596:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763343.420599:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763343.420602:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763343.420607:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21058
00000400:00000200:0.0:1787763343.420615:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763343.420620:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763343.420627:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763343.420638:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763343.420647:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763343.420652:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763343.420662:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763343.420671:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763343.420675:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763343.420684:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763343.420693:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763343.420701:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763343.420713:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763343.420736:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763343.420743:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763343.420750:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763343.420781:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763343.420788:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763343.420802:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763343.420808:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763343.420814:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763343.420823:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763343.420836:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763343.420841:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763343.420854:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763343.420861:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763343.420867:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763343.420873:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000020:00000040:0.0:1787763343.420881:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763343.420889:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763343.420894:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763343.420898:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763343.420903:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763343.420912:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763343.420931:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787763343.420943:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763343.420964:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832615424/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763343.420997:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763343.421006:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763343.421014:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832615424/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763343.421037:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763343.421049:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763343.421054:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763343.421058:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763343.421062:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763343.421066:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763343.421075:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21058
00000400:00000200:0.0:1787763343.421090:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763343.421101:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763343.421107:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763343.421118:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763343.421124:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763343.421132:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763343.421213:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763343.421224:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763343.421231:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000800:00000200:3.1:1787763343.421240:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787763343.421241:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:3.1:1787763343.421252:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000020:00000040:0.0:1787763343.421255:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763343.421267:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763343.421281:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763343.421292:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763343.421306:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763343.421314:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763343.421322:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763343.421331:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763343.421345:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763343.421353:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763343.421366:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763343.421378:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763343.421389:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763343.421394:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763343.421401:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763343.421409:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763343.421420:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763343.421433:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000020:00000040:0.0:1787763343.421442:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763343.421453:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763343.421464:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763343.421479:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763343.421482:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763343.421486:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763343.421501:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787763343.421507:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763343.421518:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832615552/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763343.421554:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763343.421564:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763343.421575:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832615552/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763343.421596:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763343.421602:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763343.421610:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763343.421619:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763343.421634:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763343.421644:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763343.421649:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763343.421659:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763343.421665:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21058
00000400:00000200:0.0:1787763343.421679:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763343.421688:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763343.421701:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763343.421715:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763343.421727:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763343.421741:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763343.421748:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763343.421780:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763343.421797:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763343.421813:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763343.421825:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763343.421839:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763343.421852:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763343.421870:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763343.421875:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763343.421883:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763343.421896:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763343.421908:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763343.421921:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763343.421932:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763343.421944:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763343.421956:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763343.421962:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763343.421971:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763343.421981:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763343.421997:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763343.422008:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763343.422020:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763343.422030:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763343.422045:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763343.422050:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763343.422064:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763343.422067:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763343.422074:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763343.422089:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787763343.422101:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763343.422121:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832615680/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763343.422196:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763343.422206:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763343.422230:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832615680/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763343.422308:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763343.422320:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21068)
00000800:00000200:3.1:1787763343.422424:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763343.422427:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787763343.422645:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763343.422647:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787763343.422913:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763343.422915:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:0.0:1787763344.060368:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763344.060371:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763344.060375:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763345.084346:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763345.084350:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763345.084353:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763347.132318:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763347.132322:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763347.132325:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763348.156391:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763348.156398:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763348.156403:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787763348.541263:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763348.541276:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787763348.542403:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763348.542406:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787763348.542870:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763348.542873:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787763348.543260:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763348.543264:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:0.0:1787763350.204414:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763350.204421:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763350.204425:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763351.228432:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763351.228438:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763351.228443:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763353.276436:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763353.276443:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763353.276448:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763353.660678:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763353.660706:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763353.660716:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763353.660734:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7931680 x1874599832616320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.660778:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763353.660788:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763353.660802:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763353.660811:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763353.660826:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763353.660841:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832616320:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763353.660861:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763353.660873:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763353.660884:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763353.660899:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763353.660916:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016000.
02000000:00000001:0.0:1787763353.660926:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763353.660949:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763353.660965:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763353.660985:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763353.661001:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832616320, portal 25
00000100:00000001:0.0:1787763353.661012:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763353.661018:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497727104 : -115097211824512 : ffff9751d7931680)
00000100:00000040:0.0:1787763353.661039:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7931680 x1874599832616320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763378 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.661075:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763353.661092:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763353.661101:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763353.661114:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832616320, offset 0
00000400:00000200:0.0:1787763353.661200:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763353.661220:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763353.661237:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763353.661251:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763353.661260:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763353.661269:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763353.661278:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763353.661291:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58050:58050:256:4294967295] 192.168.203.110@tcp LPNI seq info [58050:58050:8:4294967295]
00000400:00000200:0.0:1787763353.661304:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763353.661333:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763353.661357:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763353.661376:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763353.661396:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763353.661410:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787763353.661430:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763353.661440:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1787763353.661477:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763353.661490:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763353.661509:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763353.661547:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763353.661560:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763353.661567:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763353.661578:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763353.661594:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7931680 x1874599832616320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763378 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.661632:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763353.661650:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7931680 x1874599832616320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763378 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.661719:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763353.661728:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763353.661744:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763353.661762:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763353.661772:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315741940
00000100:00000010:0.0:1787763353.661789:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c3c0.
00000100:00000001:0.0:1787763353.661807:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763353.661823:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315741940
00000100:00000010:0.0:1787763353.661832:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c420.
00000100:00000001:0.0:1787763353.661839:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787763353.661847:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763353.661852:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763353.661857:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763353.661870:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763353/real 1787763353]  req@ffff9751d7931680 x1874599832616320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763353.661890:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763353.661893:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763353.661896:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763353.661905:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763353.661909:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763353.661915:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763353.661922:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7931680 x1874599832616320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763353.661940:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7931680 x1874599832616320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.661955:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763353.661958:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763353.661963:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763353.661965:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763353.661971:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7931680 x1874599832616320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.661981:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763353.661987:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7931680 x1874599832616320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.662005:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763353.662009:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763353.662011:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763353.662013:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763353.662016:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763353.662019:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763353.662022:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763353.662027:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763353.662030:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763353.662035:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7931680 x1874599832616320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763353.662045:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832616320:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763353.662051:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763353.662054:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763353.662058:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7931680 x1874599832616320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.662067:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763353.662070:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763353.662074:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016000.
02000000:00000001:0.0:1787763353.662078:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763353.662081:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763353.662083:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787763353.662086:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763353.662089:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9400.
02000000:00000001:0.0:1787763353.662094:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763353.662096:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763353.662098:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7931680.
00000100:00000001:0.0:1787763353.662102:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763353.662104:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763353.662106:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763353.662115:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763353.662116:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763353.662119:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763353.662121:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763353.662123:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763353.662128:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930b40 x1874599832616448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.662165:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763353.662167:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763353.662170:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763353.662173:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763353.662176:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763353.662180:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832616448:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763353.662184:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763353.662187:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763353.662190:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763353.662192:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763353.662197:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016000.
02000000:00000001:0.0:1787763353.662199:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763353.662205:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763353.662208:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763353.662213:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763353.662218:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832616448, portal 10
00000100:00000001:0.0:1787763353.662220:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763353.662222:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497724224 : -115097211827392 : ffff9751d7930b40)
00000100:00000040:0.0:1787763353.662230:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930b40 x1874599832616448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763378 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.662239:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763353.662242:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763353.662245:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763353.662248:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832616448, offset 0
00000400:00000200:0.0:1787763353.662254:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763353.662260:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763353.662266:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763353.662270:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763353.662274:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763353.662278:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763353.662281:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763353.662285:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58051:58051:256:4294967295] 192.168.203.110@tcp LPNI seq info [58051:58051:8:4294967295]
00000400:00000200:0.0:1787763353.662290:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763353.662299:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763353.662305:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763353.662311:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763353.662319:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763353.662325:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787763353.662334:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763353.662339:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1787763353.662346:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763353.662350:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763353.662356:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763353.662364:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763353.662370:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763353.662374:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763353.662379:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763353.662387:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930b40 x1874599832616448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763378 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.662405:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763353.662412:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930b40 x1874599832616448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763378 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.662423:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763353.662425:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763353.662431:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763353.662436:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:0.0:1787763353.662441:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763353.662444:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315740705
00000100:00000001:0.0:1787763353.662449:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315741940
00000100:00000001:0.0:1787763353.662453:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315740705
00000100:00000001:0.0:1787763353.662458:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315741940
00000100:00000001:0.0:1787763353.662462:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763353.662466:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763353.662471:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763353.662478:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763353/real 1787763353]  req@ffff9751d7930b40 x1874599832616448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763353.662495:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763353.662499:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763353.662503:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763353.662509:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763353.662512:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763353.662517:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763353.662524:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930b40 x1874599832616448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763353.662543:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930b40 x1874599832616448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.662553:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763353.662555:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763353.662558:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763353.662560:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763353.662565:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930b40 x1874599832616448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.662574:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763353.662579:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930b40 x1874599832616448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.662587:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763353.662590:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763353.662593:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763353.662595:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763353.662597:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763353.662613:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763353.662619:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763353.662625:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763353.662630:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763353.662638:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930b40 x1874599832616448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763353.662657:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832616448:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763353.662665:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763353.662672:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933480 x1874599832616576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.662708:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763353.662711:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763353.662715:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763353.662717:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763353.662720:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763353.662724:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832616576:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763353.662729:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763353.662732:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763353.662734:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763353.662736:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763353.662742:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015400.
02000000:00000001:0.0:1787763353.662744:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763353.662750:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763353.662753:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763353.662757:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763353.662761:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832616576, portal 10
00000100:00000001:0.0:1787763353.662764:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763353.662767:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497734784 : -115097211816832 : ffff9751d7933480)
00000100:00000040:0.0:1787763353.662775:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933480 x1874599832616576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763378 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.662788:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763353.662793:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763353.662798:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763353.662803:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832616576, offset 0
00000400:00000200:0.0:1787763353.662812:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763353.662820:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763353.662827:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763353.662834:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763353.662839:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763353.662844:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763353.662849:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763353.662856:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58052:58052:256:4294967295] 192.168.203.110@tcp LPNI seq info [58052:58052:8:4294967295]
00000400:00000200:0.0:1787763353.662865:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763353.662878:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763353.662887:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763353.662895:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763353.662900:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763353.662904:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787763353.662910:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763353.662913:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1787763353.662918:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763353.662921:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763353.662925:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763353.662930:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763353.662934:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763353.662937:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763353.662940:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763353.662945:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933480 x1874599832616576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763378 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.662956:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763353.662961:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933480 x1874599832616576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763378 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.662969:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763353.662971:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763353.662976:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763353.662980:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:0.0:1787763353.662984:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763353.662987:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315740693
00000100:00000001:0.0:1787763353.662991:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315741940
00000100:00000001:0.0:1787763353.662994:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315740693
00000100:00000001:0.0:1787763353.662997:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315741940
00000100:00000001:0.0:1787763353.663000:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763353.663002:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763353.663005:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763353.663009:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763353/real 1787763353]  req@ffff9751d7933480 x1874599832616576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763353.663019:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763353.663021:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763353.663023:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763353.663027:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763353.663029:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763353.663031:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763353.663035:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933480 x1874599832616576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763353.663045:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933480 x1874599832616576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.663053:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763353.663055:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763353.663057:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763353.663059:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763353.663064:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933480 x1874599832616576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.663073:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763353.663077:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933480 x1874599832616576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.663086:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763353.663089:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763353.663091:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763353.663092:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763353.663095:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763353.663097:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763353.663100:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763353.663103:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763353.663106:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763353.663110:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933480 x1874599832616576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763353.663120:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832616576:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763353.663124:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763353.663128:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930000 x1874599832616704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.663166:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763353.663168:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763353.663171:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763353.663174:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763353.663178:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763353.663183:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832616704:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763353.663187:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763353.663191:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763353.663194:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763353.663198:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763353.663203:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763353.663207:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763353.663212:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763353.663216:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763353.663220:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763353.663224:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832616704, portal 4
00000100:00000001:0.0:1787763353.663226:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763353.663228:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497721344 : -115097211830272 : ffff9751d7930000)
00000100:00000040:0.0:1787763353.663236:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930000 x1874599832616704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763378 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.663244:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763353.663247:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763353.663249:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763353.663252:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832616704, offset 0
00000400:00000200:0.0:1787763353.663257:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763353.663264:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763353.663268:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763353.663273:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763353.663278:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763353.663282:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763353.663286:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763353.663290:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58053:58053:256:4294967295] 192.168.203.110@tcp LPNI seq info [58053:58053:8:4294967295]
00000400:00000200:0.0:1787763353.663295:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763353.663303:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763353.663309:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763353.663315:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763353.663322:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763353.663328:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:0.1:1787763353.663337:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763353.663342:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:0.0:1787763353.663350:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763353.663354:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062500.
00000400:00000200:0.0:1787763353.663360:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763353.663367:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763353.663373:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669aa0
00000400:00000010:0.0:1787763353.663377:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669aa0.
00000100:00000001:0.0:1787763353.663382:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763353.663390:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930000 x1874599832616704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763378 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.663407:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763353.663413:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930000 x1874599832616704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763378 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.663422:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763353.663424:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763353.663430:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763353.663434:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:0.0:1787763353.663438:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763353.663441:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315740711
00000100:00000001:0.0:1787763353.663444:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315741940
00000100:00000001:0.0:1787763353.663447:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315740711
00000100:00000001:0.0:1787763353.663451:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315741940
00000100:00000001:0.0:1787763353.663455:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763353.663459:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763353.663463:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763353.663471:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763353/real 1787763353]  req@ffff9751d7930000 x1874599832616704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763353.663487:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763353.663490:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763353.663494:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763353.663499:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763353.663502:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763353.663507:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763353.663515:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930000 x1874599832616704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763353.663534:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930000 x1874599832616704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.663547:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763353.663549:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763353.663552:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763353.663554:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763353.663559:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930000 x1874599832616704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.663568:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763353.663573:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930000 x1874599832616704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.663581:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763353.663584:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763353.663586:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763353.663587:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763353.663590:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763353.663592:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763353.663595:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763353.663599:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763353.663601:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763353.663607:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930000 x1874599832616704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763353.663623:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832616704:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763353.663630:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763353.663634:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763353.663641:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930b40 x1874599832616448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.663655:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763353.663658:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763353.663662:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016000.
02000000:00000001:0.0:1787763353.663668:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763353.663671:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763353.663674:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763353.663678:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763353.663703:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb000.
02000000:00000001:0.0:1787763353.663708:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763353.663709:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763353.663711:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930b40.
00000100:00000001:0.0:1787763353.663715:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763353.663717:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763353.663720:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763353.663725:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933480 x1874599832616576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.663734:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763353.663736:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763353.663738:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015400.
02000000:00000001:0.0:1787763353.663741:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763353.663742:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763353.663744:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763353.663746:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763353.663748:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:0.0:1787763353.663750:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763353.663752:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763353.663754:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933480.
00000100:00000001:0.0:1787763353.663758:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763353.663760:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763353.663762:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763353.663765:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930000 x1874599832616704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763378 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763353.663773:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763353.663775:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763353.663777:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763353.663782:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763353.663784:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763353.663787:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787763353.663791:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763353.663794:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8400.
02000000:00000001:0.0:1787763353.663798:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763353.663800:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763353.663803:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930000.
00000100:00000001:0.0:1787763353.663807:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763353.663809:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763353.663813:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763353.663821:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763353.663824:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763353.663828:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763353.663832:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763354.300471:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763354.300477:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763354.300482:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763356.348360:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763356.348365:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763356.348368:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763357.372312:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763357.372319:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763357.372322:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763358.780734:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763358.780743:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763358.780747:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763358.780763:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930000 x1874599832616832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763358.780778:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763358.780781:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763358.780786:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763358.780790:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763358.780794:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763358.780799:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832616832:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763358.780806:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763358.780812:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763358.780815:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763358.780818:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763358.780829:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763358.780832:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763358.780841:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763358.780847:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e5d8.
00000400:00000010:0.0:1787763358.780856:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763358.780877:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832616832, portal 25
00000100:00000001:0.0:1787763358.780881:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763358.780884:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497721344 : -115097211830272 : ffff9751d7930000)
00000100:00000040:0.0:1787763358.780893:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930000 x1874599832616832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763383 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763358.780914:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763358.780918:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763358.780922:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669aa0.
00000100:00000200:0.0:1787763358.780932:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832616832, offset 0
00000400:00000200:0.0:1787763358.780954:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763358.780976:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763358.780999:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763358.781017:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763358.781029:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763358.781041:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763358.781055:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763358.781070:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58054:58054:256:4294967295] 192.168.203.110@tcp LPNI seq info [58054:58054:8:4294967295]
00000400:00000200:0.0:1787763358.781090:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763358.781130:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763358.781173:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763358.781183:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062500.
00000800:00000200:0.0:1787763358.781204:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763358.781219:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763358.781262:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763358.781279:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763358.781292:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315747060
00000100:00000010:0.0:1787763358.781314:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940cd80.
00000100:00000001:0.0:1787763358.781332:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763358.781350:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315747060
00000100:00000010:0.0:1787763358.781365:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c120.
00000100:00000001:0.0:1787763358.781396:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787763358.781410:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763358.781424:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763358.781431:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763358.781442:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763358.781457:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763358.781461:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1787763358.781468:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763358.781472:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763358.781478:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763358.781491:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933480 x1874599832616960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763358.781535:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763358.781550:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763358.781564:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763358.781575:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763358.781598:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763358.781613:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832616960:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763358.781634:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763358.781670:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763358.781679:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763358.781685:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763358.781705:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015400.
02000000:00000001:0.0:1787763358.781712:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763358.781737:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763358.781758:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e108.
00000400:00000010:0.0:1787763358.781781:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669550.
00000100:00000200:0.0:1787763358.781790:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832616960, portal 10
00000100:00000001:0.0:1787763358.781815:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763358.781830:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497734784 : -115097211816832 : ffff9751d7933480)
00000100:00000040:0.0:1787763358.781853:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933480 x1874599832616960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763383 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763358.781866:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763358.781873:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763358.781879:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669440.
00000100:00000200:0.0:1787763358.781888:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832616960, offset 0
00000400:00000200:0.0:1787763358.781911:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763358.781920:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763358.781938:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763358.781954:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763358.781964:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763358.781977:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763358.781993:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763358.782009:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58055:58055:255:4294967295] 192.168.203.110@tcp LPNI seq info [58055:58055:7:4294967295]
00000400:00000200:0.0:1787763358.782046:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763358.782077:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763358.782108:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763358.782119:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062b00.
00000800:00000200:0.0:1787763358.782178:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763358.782199:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763358.782208:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763358.782217:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:0.0:1787763358.782227:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763358.782232:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315745825
00000100:00000001:0.0:1787763358.782250:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315747060
00000100:00000001:0.0:1787763358.782257:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315745825
00000100:00000001:0.0:1787763358.782269:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315747060
00000100:00000001:0.0:1787763358.782278:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763358.782292:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763358.782304:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763358.782313:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763358.782328:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763358.782335:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1787763358.782349:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763358.782359:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763358.782367:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763358.782386:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930b40 x1874599832617088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763358.782432:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763358.782442:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763358.782453:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763358.782462:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763358.782475:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763358.782495:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832617088:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763358.782519:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763358.782529:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763358.782538:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763358.782547:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763358.782560:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016000.
02000000:00000001:0.0:1787763358.782570:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763358.782590:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763358.782602:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e268.
00000400:00000010:0.0:1787763358.782616:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669bb0.
00000100:00000200:0.0:1787763358.782630:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832617088, portal 10
00000100:00000001:0.0:1787763358.782641:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763358.782676:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497724224 : -115097211827392 : ffff9751d7930b40)
00000100:00000040:0.0:1787763358.782705:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930b40 x1874599832617088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763383 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763358.782753:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763358.782767:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763358.782782:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669990.
00000100:00000200:0.0:1787763358.782790:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832617088, offset 0
00000400:00000200:0.0:1787763358.782804:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763358.782814:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763358.782824:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763358.782831:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763358.782835:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763358.782840:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763358.782845:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763358.782851:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58056:58056:254:4294967295] 192.168.203.110@tcp LPNI seq info [58056:58056:6:4294967295]
00000400:00000200:0.0:1787763358.782859:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763358.782873:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763358.782882:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763358.782890:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062000.
00000800:00000200:0.0:1787763358.782898:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763358.782905:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763358.782912:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763358.782919:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:0.0:1787763358.782926:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763358.782930:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315745813
00000100:00000001:0.0:1787763358.782935:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315747060
00000100:00000001:0.0:1787763358.782939:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315745813
00000100:00000001:0.0:1787763358.782943:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315747060
00000100:00000001:0.0:1787763358.782948:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763358.782953:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763358.782959:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763358.782963:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763358.782970:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763358.782974:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1787763358.782978:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763358.782982:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763358.782985:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763358.782995:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7931680 x1874599832617216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763358.783011:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763358.783015:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763358.783019:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763358.783023:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763358.783028:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763358.783032:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832617216:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763358.783037:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763358.783040:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763358.783043:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763358.783045:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763358.783052:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763358.783054:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763358.783061:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763358.783066:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e528.
00000400:00000010:0.0:1787763358.783074:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669c38.
00000100:00000200:0.0:1787763358.783080:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832617216, portal 4
00000100:00000001:0.0:1787763358.783084:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763358.783087:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497727104 : -115097211824512 : ffff9751d7931680)
00000100:00000040:0.0:1787763358.783098:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7931680 x1874599832617216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763383 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763358.783114:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763358.783119:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763358.783123:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669a18.
00000100:00000200:0.0:1787763358.783128:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832617216, offset 0
00000400:00000200:0.0:1787763358.783199:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763358.783208:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763358.783216:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763358.783222:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763358.783227:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763358.783231:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763358.783236:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763358.783243:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58057:58057:253:4294967295] 192.168.203.110@tcp LPNI seq info [58057:58057:5:4294967295]
00000400:00000200:0.0:1787763358.783252:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763358.783266:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763358.783274:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763358.783282:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062700.
00000800:00000200:0.0:1787763358.783290:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763358.783299:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763358.783304:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763358.783311:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:0.0:1787763358.783318:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763358.783322:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315745831
00000100:00000001:0.0:1787763358.783327:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315747060
00000100:00000001:0.0:1787763358.783332:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315745831
00000100:00000001:0.0:1787763358.783336:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315747060
00000100:00000001:0.0:1787763358.783340:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763358.783344:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763358.783349:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763358.783353:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763358.783359:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763358.783362:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1787763358.783366:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763358.783369:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763358.783372:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763358.783376:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763359.420483:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763359.420489:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763359.420494:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763360.444396:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763360.444402:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763360.444407:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763361.853168:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763361.853178:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763361.853185:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763361.853205:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763358/real 1787763361]  req@ffff9751d7930b40 x1874599832617088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763383 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763361.853231:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763361.853240:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763361.853251:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e268.
00000400:00000200:0.0:1787763361.853268:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669bb0
00000400:00000010:0.0:1787763361.853279:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669bb0.
00000100:00000001:0.0:1787763361.853294:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763361.853312:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930b40 x1874599832617088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763361.853358:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930b40 x1874599832617088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763361.853392:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763361.853400:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763361.853410:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763361.853417:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763361.853437:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930b40 x1874599832617088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763361.853471:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763361.853494:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930b40 x1874599832617088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763361.853538:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763361.853545:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763361.853549:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763361.853552:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763361.853557:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763361.853561:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763361.853580:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763361.853598:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763361.853605:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763361.853621:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930b40 x1874599832617088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763361.853675:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832617088:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763361.853685:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763361.853694:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763358/real 1787763361]  req@ffff9751d7933480 x1874599832616960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763361.853713:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763361.853715:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763361.853719:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e108.
00000400:00000200:0.0:1787763361.853734:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669550
00000400:00000010:0.0:1787763361.853737:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669550.
00000100:00000001:0.0:1787763361.853741:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763361.853746:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933480 x1874599832616960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763361.853757:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933480 x1874599832616960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763361.853766:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763361.853767:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763361.853770:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763361.853772:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763361.853776:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933480 x1874599832616960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763361.853786:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763361.853791:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933480 x1874599832616960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763361.853799:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763361.853803:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763361.853805:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763361.853807:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763361.853810:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763361.853823:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763361.853827:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763361.853985:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763361.853991:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763361.854002:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933480 x1874599832616960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763361.854021:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832616960:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763361.854029:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763361.854038:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763358/real 1787763361]  req@ffff9751d7930000 x1874599832616832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763361.854056:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763361.854059:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763361.854064:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e5d8.
00000400:00000200:0.0:1787763361.854070:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763361.854074:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763361.854080:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763361.854087:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930000 x1874599832616832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763361.854104:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930000 x1874599832616832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763361.854119:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763361.854121:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763361.854125:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763361.854129:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763361.854226:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930000 x1874599832616832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763361.854244:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763361.854255:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930000 x1874599832616832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763361.854270:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763361.854277:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763361.854282:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763361.854285:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763361.854288:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763361.854291:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763361.854296:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763361.854304:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763361.854309:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763361.854318:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930000 x1874599832616832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763361.854334:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832616832:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763361.854343:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763361.854348:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763361.854355:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930b40 x1874599832617088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763361.854370:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763361.854375:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763361.854381:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016000.
02000000:00000001:0.0:1787763361.854390:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763361.854395:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763361.854398:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763361.854401:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763361.854405:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8000.
02000000:00000001:0.0:1787763361.854409:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763361.854413:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763361.854416:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930b40.
00000100:00000001:0.0:1787763361.854420:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763361.854422:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763361.854424:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763361.854429:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933480 x1874599832616960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763361.854438:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763361.854439:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763361.854442:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015400.
02000000:00000001:0.0:1787763361.854444:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763361.854446:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763361.854448:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763361.854450:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763361.854452:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb400.
02000000:00000001:0.0:1787763361.854455:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763361.854456:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763361.854458:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933480.
00000100:00000001:0.0:1787763361.854462:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763361.854463:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763361.854465:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763361.854469:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930000 x1874599832616832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763361.854476:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763361.854478:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763361.854479:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763361.854482:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763361.854483:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763361.854484:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787763361.854487:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763361.854488:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca000.
02000000:00000001:0.0:1787763361.854491:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763361.854492:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763361.854494:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930000.
00000100:00000001:0.0:1787763361.854497:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763361.854498:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763361.854502:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763361.854505:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763361.854507:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763361.854509:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763361.854516:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763358/real 1787763361]  req@ffff9751d7931680 x1874599832617216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763361.854534:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763361.854537:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763361.854541:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e528.
00000400:00000200:0.0:1787763361.854547:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669c38
00000400:00000010:0.0:1787763361.854551:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669c38.
00000100:00000001:0.0:1787763361.854557:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763361.854563:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7931680 x1874599832617216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763361.854582:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7931680 x1874599832617216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763361.854596:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763361.854599:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763361.854603:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763361.854606:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763361.854613:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7931680 x1874599832617216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763361.854649:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763361.854658:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7931680 x1874599832617216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763361.854667:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763361.854671:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763361.854673:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763361.854674:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763361.854677:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763361.854679:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763361.854684:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763361.854689:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763361.854692:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763361.854697:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7931680 x1874599832617216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763361.854706:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832617216:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763361.854711:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763361.854714:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763361.854717:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7931680 x1874599832617216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763383 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763361.854725:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763361.854727:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763361.854729:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763361.854733:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763361.854734:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763361.854736:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787763361.854739:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763361.854741:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9000.
02000000:00000001:0.0:1787763361.854744:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763361.854746:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763361.854748:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7931680.
00000100:00000001:0.0:1787763361.854752:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763361.854753:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763361.854756:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763361.854764:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763361.854766:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763361.854769:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763361.854772:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763362.492439:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763362.492447:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763362.492454:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763363.516364:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763363.516370:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763363.516374:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763363.900286:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763363.900296:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763363.900298:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763363.900304:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763363.900309:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763363.900313:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.900316:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763363.900319:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.900322:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763363.900798:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763363.900808:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763363.900814:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763363.900836:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832617344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.900887:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763363.900895:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763363.900906:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763363.900912:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763363.900925:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763363.900940:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832617344:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763363.900958:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763363.900969:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763363.900978:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763363.900994:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763363.901010:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763363.901016:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763363.901034:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763363.901041:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e528.
00000400:00000010:0.0:1787763363.901054:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669c38.
00000100:00000200:0.0:1787763363.901069:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832617344, portal 25
00000100:00000001:0.0:1787763363.901077:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763363.901082:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:0.0:1787763363.901099:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832617344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763388 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.901128:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763363.901210:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763363.901220:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763363.901232:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832617344, offset 0
00000400:00000200:0.0:1787763363.901252:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763363.901273:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763363.901292:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763363.901309:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763363.901323:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763363.901336:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763363.901357:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763363.901375:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58058:58058:256:4294967295] 192.168.203.110@tcp LPNI seq info [58058:58058:8:4294967295]
00000400:00000200:0.0:1787763363.901403:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763363.901436:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763363.901460:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763363.901471:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062200.
00000800:00000200:0.0:1787763363.901489:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763363.901501:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787763363.901537:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763363.901548:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787763363.901570:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763363.901578:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062200.
00000400:00000200:0.0:1787763363.901592:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763363.901603:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763363.901635:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763363.901640:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763363.901649:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763363.901664:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832617344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763388 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.901705:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763363.901721:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832617344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763388 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.901771:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763363.901779:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763363.901794:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763363.901814:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763363.901825:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315752180
00000100:00000010:0.0:1787763363.901840:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c780.
00000100:00000001:0.0:1787763363.901858:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763363.901880:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315752180
00000100:00000010:0.0:1787763363.901891:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c060.
00000100:00000001:0.0:1787763363.901901:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787763363.901914:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763363.901922:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763363.901926:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763363.901937:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763363/real 1787763363]  req@ffff9751f86583c0 x1874599832617344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763363.901949:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763363.901953:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763363.901962:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e528.
00000400:00000200:0.0:1787763363.901972:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669c38
00000400:00000010:0.0:1787763363.901977:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669c38.
00000100:00000001:0.0:1787763363.901981:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763363.901990:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832617344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763363.902016:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832617344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.902031:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763363.902038:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.902046:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763363.902052:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763363.902072:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832617344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.902099:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763363.902114:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832617344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.902164:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763363.902167:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.902169:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763363.902170:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.902173:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763363.902177:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763363.902182:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763363.902193:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763363.902197:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763363.902208:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832617344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763363.902220:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832617344:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763363.902233:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.902243:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763363.902251:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832617344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.902271:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763363.902276:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763363.902282:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763363.902289:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763363.902294:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763363.902298:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787763363.902308:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763363.902312:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:0.0:1787763363.902320:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763363.902325:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763363.902332:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:0.0:1787763363.902342:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763363.902347:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.902350:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.902367:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763363.902373:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763363.902382:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763363.902390:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763363.902400:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763363.902417:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832617472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.902462:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763363.902471:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763363.902483:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763363.902492:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763363.902509:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763363.902523:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832617472:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763363.902543:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763363.902548:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763363.902561:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763363.902568:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763363.902575:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763363.902579:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763363.902593:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763363.902598:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e528.
00000400:00000010:0.0:1787763363.902606:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669c38.
00000100:00000200:0.0:1787763363.902626:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832617472, portal 10
00000100:00000001:0.0:1787763363.902629:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763363.902632:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000040:0.0:1787763363.902650:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832617472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763388 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.902670:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763363.902680:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763363.902689:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763363.902704:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832617472, offset 0
00000400:00000200:0.0:1787763363.902718:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763363.902739:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763363.902751:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763363.902757:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763363.902770:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763363.902777:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763363.902784:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763363.902799:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58059:58059:256:4294967295] 192.168.203.110@tcp LPNI seq info [58059:58059:8:4294967295]
00000400:00000200:0.0:1787763363.902824:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763363.902848:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763363.902857:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763363.902866:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062200.
00000800:00000200:0.0:1787763363.902874:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763363.902876:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787763363.902883:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763363.902886:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787763363.902890:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763363.902892:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062200.
00000400:00000200:0.0:1787763363.902897:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763363.902901:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763363.902904:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763363.902905:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763363.902907:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763363.902911:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832617472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763388 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.902918:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763363.902920:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832617472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763388 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.902925:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763363.902926:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763363.902931:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763363.902940:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:0.0:1787763363.902943:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763363.902945:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315750945
00000100:00000001:0.0:1787763363.902947:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315752180
00000100:00000001:0.0:1787763363.902949:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315750945
00000100:00000001:0.0:1787763363.902951:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315752180
00000100:00000001:0.0:1787763363.902952:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763363.902954:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763363.902955:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763363.902958:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763363/real 1787763363]  req@ffff9751f8658780 x1874599832617472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763363.902963:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763363.902964:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763363.902966:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e528.
00000400:00000200:0.0:1787763363.902968:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669c38
00000400:00000010:0.0:1787763363.902969:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669c38.
00000100:00000001:0.0:1787763363.902971:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763363.902973:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832617472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763363.902979:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832617472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.902983:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763363.902984:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.902986:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763363.902987:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763363.902989:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832617472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.902993:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763363.902995:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832617472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.903001:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763363.903003:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.903004:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763363.903005:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.903006:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763363.903019:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763363.903021:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763363.903023:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763363.903025:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763363.903027:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832617472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763363.903034:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832617472:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763363.903038:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763363.903043:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832617600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.903051:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763363.903052:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763363.903055:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763363.903058:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763363.903060:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763363.903064:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832617600:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763363.903068:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763363.903071:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763363.903073:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763363.903075:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763363.903079:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763363.903081:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763363.903084:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763363.903086:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e528.
00000400:00000010:0.0:1787763363.903088:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669c38.
00000100:00000200:0.0:1787763363.903091:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832617600, portal 10
00000100:00000001:0.0:1787763363.903093:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763363.903094:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:0.0:1787763363.903099:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832617600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763388 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.903104:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763363.903106:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763363.903107:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763363.903109:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832617600, offset 0
00000400:00000200:0.0:1787763363.903112:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763363.903114:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763363.903116:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763363.903118:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763363.903120:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763363.903122:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763363.903124:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763363.903126:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58060:58060:256:4294967295] 192.168.203.110@tcp LPNI seq info [58060:58060:8:4294967295]
00000400:00000200:0.0:1787763363.903129:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763363.903152:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763363.903156:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763363.903158:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062200.
00000800:00000200:0.0:1787763363.903161:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763363.903163:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787763363.903166:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763363.903168:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787763363.903171:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763363.903173:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062200.
00000400:00000200:0.0:1787763363.903175:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763363.903177:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763363.903180:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763363.903182:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763363.903183:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763363.903186:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832617600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763388 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.903192:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763363.903194:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832617600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763388 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.903199:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763363.903200:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763363.903202:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763363.903205:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:0.0:1787763363.903207:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763363.903209:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315750934
00000100:00000001:0.0:1787763363.903211:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315752180
00000100:00000001:0.0:1787763363.903212:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315750934
00000100:00000001:0.0:1787763363.903214:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315752180
00000100:00000001:0.0:1787763363.903215:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763363.903216:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763363.903217:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763363.903219:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763363/real 1787763363]  req@ffff9751f8659a40 x1874599832617600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763363.903225:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763363.903226:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763363.903227:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e528.
00000400:00000200:0.0:1787763363.903229:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669c38
00000400:00000010:0.0:1787763363.903231:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669c38.
00000100:00000001:0.0:1787763363.903232:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763363.903234:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832617600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763363.903240:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832617600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.903245:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763363.903246:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.903247:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763363.903248:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763363.903251:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832617600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.903255:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763363.903257:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832617600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.903261:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763363.903262:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.903263:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763363.903264:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.903265:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763363.903266:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763363.903268:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763363.903270:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763363.903271:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763363.903273:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832617600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763363.903278:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832617600:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763363.903280:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.903281:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763363.903284:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832617472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.903288:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763363.903290:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763363.903291:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015800.
02000000:00000001:0.0:1787763363.903293:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763363.903295:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763363.903296:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763363.903298:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763363.903299:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:0.0:1787763363.903301:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763363.903302:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763363.903304:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:0.0:1787763363.903306:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763363.903307:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.903308:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763363.903310:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832617600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.903314:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763363.903314:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763363.903315:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763363.903317:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763363.903318:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763363.903319:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787763363.903320:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763363.903321:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:0.0:1787763363.903322:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763363.903323:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763363.903324:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:0.0:1787763363.903325:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763363.903326:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.903327:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.903331:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763363.903333:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763363.903334:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763363.903336:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763363.903337:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763363.903340:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832617728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.903345:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763363.903347:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763363.903348:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763363.903350:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763363.903351:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763363.903353:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832617728:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763363.903355:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763363.903356:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763363.903358:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763363.903359:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763363.903360:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763363.903362:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763363.903364:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787763363.903366:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e528.
00000400:00000010:0.0:1787763363.903369:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669c38.
00000100:00000200:0.0:1787763363.903371:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832617728, portal 4
00000100:00000001:0.0:1787763363.903372:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763363.903373:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:0.0:1787763363.903378:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832617728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763388 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.903386:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763363.903388:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787763363.903391:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787763363.903393:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832617728, offset 0
00000400:00000200:0.0:1787763363.903398:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763363.903401:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763363.903405:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787763363.903408:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763363.903411:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763363.903414:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763363.903417:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787763363.903420:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58061:58061:256:4294967295] 192.168.203.110@tcp LPNI seq info [58061:58061:8:4294967295]
00000400:00000200:0.0:1787763363.903425:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763363.903432:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787763363.903437:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763363.903440:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062200.
00000800:00000200:0.0:1787763363.903443:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787763363.903445:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:0.1:1787763363.903448:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787763363.903450:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:0.0:1787763363.903453:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787763363.903455:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062200.
00000400:00000200:0.0:1787763363.903457:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763363.903459:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763363.903462:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787763363.903463:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787763363.903465:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763363.903469:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832617728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763388 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.903478:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763363.903482:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832617728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763388 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.903489:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763363.903491:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763363.903495:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763363.903498:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:0.0:1787763363.903503:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763363.903505:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315750951
00000100:00000001:0.0:1787763363.903508:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315752180
00000100:00000001:0.0:1787763363.903510:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315750951
00000100:00000001:0.0:1787763363.903513:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315752180
00000100:00000001:0.0:1787763363.903515:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763363.903517:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763363.903518:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787763363.903522:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763363/real 1787763363]  req@ffff9751f8658000 x1874599832617728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787763363.903528:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787763363.903529:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787763363.903531:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e528.
00000400:00000200:0.0:1787763363.903533:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669c38
00000400:00000010:0.0:1787763363.903534:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669c38.
00000100:00000001:0.0:1787763363.903536:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787763363.903538:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832617728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763363.903544:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832617728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.903548:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763363.903549:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.903550:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763363.903551:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763363.903553:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832617728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.903558:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763363.903561:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832617728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.903566:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763363.903568:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.903569:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763363.903569:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.903570:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763363.903571:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787763363.903573:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787763363.903575:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763363.903576:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763363.903580:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832617728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763363.903585:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832617728:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787763363.903588:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.903589:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763363.903591:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832617728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763388 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763363.903594:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787763363.903595:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787763363.903597:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016c00.
02000000:00000001:0.0:1787763363.903598:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787763363.903599:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787763363.903600:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787763363.903601:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787763363.903602:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:0.0:1787763363.903604:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787763363.903604:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787763363.903605:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:0.0:1787763363.903607:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763363.903608:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.903609:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763363.903623:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763363.903624:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763363.903625:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763363.903627:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763364.924227:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763364.924241:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763364.924245:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763364.924254:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763364.924260:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763364.924266:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763364.924269:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763364.924272:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763364.924276:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763365.564436:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763365.564442:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763365.564447:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763365.948284:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763365.948294:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763365.948298:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763365.948306:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763365.948313:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763365.948318:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763365.948322:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763365.948326:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763365.948339:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763366.588484:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763366.588490:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763366.588496:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763368.636465:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763368.636474:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763368.636481:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763369.020307:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763369.020316:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763369.020320:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763369.020324:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763369.020330:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763369.020338:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21083
00000400:00000200:0.0:1787763369.020348:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763369.020357:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763369.020363:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763369.020367:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763369.020373:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763369.020376:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763369.020380:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763369.020387:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763369.020389:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763369.020393:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763369.020399:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763369.020404:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763369.020408:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763369.020413:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763369.020416:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763369.020420:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763369.020423:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763369.020427:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763369.020430:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763369.020433:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763369.020437:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763369.020440:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763369.020443:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763369.020446:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763369.020448:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763369.020459:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763369.020462:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763369.020467:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763369.020468:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763369.020472:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763369.020474:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763369.020479:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763369.020487:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787763369.020495:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763369.020505:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832617856/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763369.020517:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763369.020521:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763369.020526:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832617856/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763369.020554:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763369.020557:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763369.020562:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763369.020564:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763369.020565:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763369.020569:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763369.020573:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21083
00000400:00000200:0.0:1787763369.020596:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763369.020600:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763369.020604:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763369.020612:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763369.020618:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763369.020625:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763369.020646:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763369.020651:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763369.020656:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763369.020662:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763369.020668:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763369.020676:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763369.020681:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763369.020691:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763369.020698:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763369.020707:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763369.020713:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763369.020723:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763369.020734:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763369.020740:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763369.020747:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763369.020761:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763369.020770:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763369.020779:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763369.020790:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763369.020804:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763369.020811:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763369.020824:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763369.020835:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763369.020847:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763369.020852:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763369.020858:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763369.020864:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763369.020868:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763369.020878:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787763369.020887:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763369.020920:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832617984/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763369.020933:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763369.020937:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763369.020946:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832617984/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763369.020971:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763369.020978:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763369.020992:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763369.020997:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763369.021005:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763369.021013:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763369.021024:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21083
00000400:00000200:0.0:1787763369.021041:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763369.021053:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763369.021060:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763369.021068:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763369.021072:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763369.021077:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763369.021081:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763369.021086:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763369.021091:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763369.021095:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763369.021100:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763369.021104:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763369.021111:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763369.021119:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763369.021125:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763369.021131:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763369.021136:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763369.021139:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763369.021143:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763369.021152:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763369.021154:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763369.021158:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:3.1:1787763369.021208:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763369.021212:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
02000000:00000001:0.0:1787763369.021219:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763369.021226:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763369.021230:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763369.021237:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763369.021243:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763369.021254:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763369.021264:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763369.021271:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763369.021273:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763369.021278:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763369.021281:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763369.021286:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763369.021293:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787763369.021302:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763369.021313:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832618112/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763369.021371:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763369.021380:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763369.021397:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832618112/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763369.021422:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763369.021432:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763369.021441:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763369.021449:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763369.021455:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763369.021459:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763369.021463:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763369.021472:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763369.021481:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21083
00000400:00000200:0.0:1787763369.021491:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763369.021499:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763369.021506:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763369.021517:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763369.021524:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763369.021532:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763369.021541:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763369.021549:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763369.021553:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763369.021557:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763369.021562:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763369.021570:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763369.021597:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763369.021625:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763369.021632:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763369.021641:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763369.021650:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763369.021658:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763369.021668:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763369.021679:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763369.021686:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763369.021694:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763369.021702:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763369.021708:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763369.021714:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763369.021725:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763369.021732:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763369.021741:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763369.021748:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763369.021754:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763369.021762:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763369.021775:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763369.021783:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763369.021792:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763369.021808:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787763369.021822:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763369.021846:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832618240/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763369.021876:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763369.021883:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763369.021892:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832618240/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763369.021924:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763369.021932:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21093)
00000800:00000200:3.1:1787763369.022181:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763369.022184:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787763369.022484:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763369.022487:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787763369.022842:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763369.022845:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:0.0:1787763369.660442:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763369.660448:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763369.660454:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763371.708413:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763371.708419:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763371.708424:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763372.732383:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763372.732392:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763372.732398:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763373.756326:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763373.756332:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763373.756337:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763374.140333:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763374.140345:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763374.140352:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763374.140360:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763374.140366:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763374.140377:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21088
00000400:00000200:0.0:1787763374.140387:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763374.140396:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763374.140401:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763374.140405:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763374.140409:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763374.140414:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763374.140420:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763374.140427:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763374.140431:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763374.140435:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763374.140441:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763374.140445:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763374.140449:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763374.140464:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763374.140468:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763374.140470:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763374.140473:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763374.140477:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763374.140480:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763374.140483:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763374.140488:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763374.140494:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763374.140500:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763374.140503:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763374.140507:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763374.140532:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
00000020:00000040:0.0:1787763374.140538:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763374.140555:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763374.140557:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763374.140562:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763374.140566:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763374.140572:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763374.140584:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763374.140592:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763374.140605:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832618368/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763374.140615:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763374.140619:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763374.140624:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832618368/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763374.140657:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763374.140660:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763374.140664:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763374.140667:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763374.140670:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763374.140673:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763374.140677:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21088
00000400:00000200:0.0:1787763374.140683:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763374.140686:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763374.140688:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763374.140691:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763374.140693:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763374.140696:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763374.140699:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763374.140702:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763374.140703:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763374.140705:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763374.140709:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763374.140717:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763374.140721:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763374.140732:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763374.140735:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763374.140737:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763374.140739:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763374.140745:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763374.140750:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763374.140761:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763374.140767:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763374.140784:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763374.140800:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763374.140808:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763374.140815:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763374.140818:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763374.140831:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763374.140836:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787763374.140845:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763374.140859:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763374.140867:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763374.140875:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763374.140883:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763374.140902:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763374.140918:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787763374.140937:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763374.140952:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832618496/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763374.140972:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763374.140979:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763374.141000:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832618496/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763374.141026:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763374.141032:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763374.141043:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763374.141059:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763374.141069:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763374.141079:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763374.141095:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21088
00000400:00000200:0.0:1787763374.141113:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763374.141126:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763374.141161:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763374.141169:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763374.141183:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763374.141189:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763374.141197:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763374.141208:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763374.141211:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763374.141221:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763374.141226:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763374.141238:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763374.141249:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763374.141262:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763374.141270:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763374.141282:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763374.141292:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763374.141301:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763374.141311:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763374.141322:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763374.141329:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763374.141337:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763374.141353:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763374.141365:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763374.141375:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763374.141380:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763374.141383:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763374.141391:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000020:00000040:0.0:1787763374.141402:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763374.141410:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763374.141413:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763374.141423:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763374.141427:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:2.1:1787763374.141433:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:0.0:1787763374.141434:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787763374.141447:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
02000000:00000010:0.0:1787763374.141448:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763374.141466:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763374.141486:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832618624/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763374.141511:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763374.141521:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763374.141533:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832618624/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763374.141561:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763374.141567:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763374.141576:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763374.141581:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763374.141601:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763374.141607:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763374.141614:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763374.141624:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763374.141635:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21088
00000400:00000200:0.0:1787763374.141657:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763374.141676:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763374.141688:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763374.141700:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763374.141707:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763374.141723:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763374.141735:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763374.141747:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763374.141758:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763374.141766:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763374.141771:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763374.141786:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763374.141789:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763374.141794:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763374.141802:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763374.141806:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763374.141810:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763374.141819:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763374.141841:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763374.141852:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763374.141858:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763374.141864:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763374.141875:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763374.141882:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763374.141886:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763374.141899:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763374.141911:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763374.141923:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763374.141932:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763374.141949:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763374.141956:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763374.141964:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763374.141970:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763374.141975:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763374.141989:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763374.142001:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763374.142026:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832618752/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763374.142052:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763374.142059:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763374.142083:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832618752/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763374.142125:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763374.142171:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21098)
00000800:00000200:2.1:1787763374.142487:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763374.142491:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787763374.142896:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763374.142900:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787763374.143310:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763374.143314:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:0.0:1787763374.780504:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763374.780510:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763374.780514:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763375.804337:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763375.804341:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763375.804349:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763376.828355:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763376.828359:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763376.828363:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763377.852391:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763377.852396:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763377.852400:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763378.876362:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763378.876368:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763378.876372:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1787763379.261068:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763379.261078:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787763379.261713:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763379.261716:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787763379.262135:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763379.262136:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787763379.262490:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763379.262492:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:0.0:1787763379.900314:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763379.900321:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763379.900327:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763380.924378:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763380.924383:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763380.924387:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763381.948420:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763381.948427:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763381.948433:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763382.972418:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763382.972423:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763382.972438:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763383.996283:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763383.996287:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763383.996289:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1787763384.380824:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763384.380829:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787763384.381303:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763384.381305:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787763384.381510:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763384.381512:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787763384.381718:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763384.381720:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000020:00000001:0.0:1787763385.020291:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763385.020294:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763385.020297:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763387.068334:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763387.068339:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763387.068343:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763388.092276:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763388.092280:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763388.092284:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763389.116334:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763389.116339:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763389.116342:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1787763389.500880:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763389.500887:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787763389.501358:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763389.501360:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787763389.501634:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763389.501637:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787763389.501838:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763389.501840:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:0.0:1787763390.140369:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763390.140374:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763390.140377:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763391.164350:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763391.164356:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763391.164361:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763392.188440:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763392.188447:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763392.188452:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763393.212445:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763393.212451:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763393.212456:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763394.236459:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763394.236465:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763394.236470:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763394.620313:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763394.620321:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763394.620327:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763394.620336:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763394.620342:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763394.620349:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763394.620356:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763394.620363:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763394.620373:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787763394.621325:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763394.621338:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787763394.622494:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763394.622496:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787763394.622705:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763394.622707:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787763394.622896:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763394.622898:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:0.0:1787763395.260386:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763395.260415:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763395.260422:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763395.644238:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763395.644245:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763395.644247:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763395.644252:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763395.644255:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763395.644259:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763395.644263:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763395.644268:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763395.644276:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763396.284334:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763396.284339:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763396.284342:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763396.668263:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763396.668276:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763396.668291:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763396.668301:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763396.668308:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763396.668314:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763396.668319:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763396.668325:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763396.668330:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763397.692292:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763397.692305:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763397.692309:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763397.692317:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763397.692323:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763397.692328:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763397.692332:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763397.692336:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763397.692340:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763398.332336:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763398.332341:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763398.332345:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763398.716302:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763398.716313:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763398.716317:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763398.716325:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763398.716332:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763398.716338:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763398.716342:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763398.716347:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763398.716352:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763399.356537:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763399.356543:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763399.356548:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763399.740277:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763399.740285:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763399.740291:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763399.740303:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763399.740317:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763399.740327:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763399.740340:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763399.740351:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763399.740385:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787763399.741475:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763399.741487:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787763399.742925:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763399.742931:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787763399.743463:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763399.743469:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787763399.743951:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763399.743955:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:0.0:1787763400.764249:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763400.764259:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763400.764261:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763400.764267:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763400.764272:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763400.764277:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763400.764280:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763400.764284:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763400.764287:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763401.404396:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763401.404401:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763401.404404:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763401.788224:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763401.788235:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763401.788238:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763401.788247:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763401.788253:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763401.788258:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763401.788261:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763401.788264:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763401.788267:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763402.428339:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763402.428344:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763402.428361:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763402.812236:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763402.812257:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763402.812260:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763402.812268:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763402.812273:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763402.812279:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763402.812283:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763402.812288:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763402.812293:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763403.836233:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763403.836243:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763403.836245:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763403.836251:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763403.836256:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763403.836260:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763403.836262:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763403.836266:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763403.836268:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763404.476383:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763404.476387:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763404.476390:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763404.860289:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763404.860304:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763404.860311:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763404.860321:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763404.860349:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763404.860356:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763404.860361:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763404.860367:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763404.860373:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763404.862397:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763404.862401:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787763404.863459:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763404.863466:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787763404.864025:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763404.864027:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787763404.864317:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763404.864319:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:0.0:1787763405.500374:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763405.500378:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763405.500381:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763405.884244:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763405.884253:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763405.884256:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763405.884263:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763405.884267:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763405.884271:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763405.884273:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763405.884276:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763405.884279:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763406.908255:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763406.908265:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763406.908268:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763406.908274:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763406.908279:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763406.908282:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763406.908285:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763406.908288:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763406.908290:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763407.548437:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763407.548441:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763407.548444:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763407.932231:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763407.932242:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763407.932244:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763407.932250:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763407.932253:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763407.932257:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763407.932260:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763407.932262:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763407.932264:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763408.572326:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763408.572330:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763408.572333:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763408.956292:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763408.956316:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763408.956318:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763408.956324:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763408.956327:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763408.956331:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763408.956334:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763408.956337:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763408.956339:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763409.980272:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763409.980283:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763409.980287:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763409.980318:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763409.980328:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763409.980338:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763409.980348:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763409.980354:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763409.980364:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787763409.981216:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763409.981221:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787763409.982769:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763409.982772:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787763409.983224:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763409.983228:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787763409.983783:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763409.983787:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:0.0:1787763410.620474:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763410.620482:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763410.620489:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763411.004428:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763411.004438:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763411.004442:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763411.004450:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763411.004455:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763411.004460:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763411.004464:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763411.004469:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763411.004474:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763411.644475:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763411.644481:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763411.644484:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763412.668395:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763412.669669:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763412.669675:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763413.692391:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763413.692396:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763413.692399:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763414.716492:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763414.716496:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763414.716499:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763415.100356:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763415.100365:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763415.100369:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763415.100377:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763415.100382:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763415.100389:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763415.100396:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763415.100399:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763415.100403:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787763415.101001:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763415.101006:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787763415.102369:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763415.102372:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787763415.102853:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763415.102859:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787763415.103634:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763415.103638:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:0.0:1787763415.740412:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763415.740417:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763415.740421:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763416.124323:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763416.124333:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763416.124336:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763416.124343:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763416.124348:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763416.124352:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763416.124356:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763416.124360:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763416.124363:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763416.764403:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763416.764407:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763416.764410:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763417.148262:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763417.148274:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763417.148278:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763417.148290:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763417.148295:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763417.148299:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763417.148304:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763417.148306:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763417.148310:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763417.788430:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763417.788437:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763417.788441:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763418.172309:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763418.172320:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763418.172324:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763418.172333:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763418.172340:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763418.172345:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763418.172350:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763418.172355:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763418.172360:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763418.812450:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763418.812459:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763418.812466:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763419.196261:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763419.196270:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763419.196273:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763419.196281:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763419.196287:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763419.196292:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763419.196296:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763419.196300:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763419.196306:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763419.836474:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763419.836480:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763419.836485:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763420.220342:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763420.220353:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763420.220357:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763420.220367:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763420.220378:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763420.220390:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763420.220401:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763420.220412:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763420.220434:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787763420.221842:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763420.221859:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787763420.222697:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763420.222701:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787763420.223202:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763420.223206:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787763420.223594:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763420.223598:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000020:00000001:0.0:1787763420.860446:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763420.860452:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763420.860456:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763421.244285:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763421.244294:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763421.244298:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763421.244305:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763421.244310:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763421.244315:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763421.244318:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763421.244322:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763421.244326:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763421.884572:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763421.884578:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763421.884583:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763422.268262:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763422.268272:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763422.268275:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763422.268282:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763422.268288:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763422.268294:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763422.268299:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763422.268303:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763422.268307:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763422.908474:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763422.908480:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763422.908486:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763423.292258:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763423.292271:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763423.292275:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763423.292286:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763423.292293:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763423.292299:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763423.292304:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763423.292309:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763423.292314:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763423.932395:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763423.932403:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763423.932408:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763424.316302:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763424.316315:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763424.316319:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763424.316328:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763424.316334:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763424.316340:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763424.316346:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763424.316351:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763424.316358:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763424.956436:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763424.956442:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763424.956447:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763425.340292:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763425.340302:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763425.340305:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763425.340312:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763425.340317:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763425.340322:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763425.340326:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763425.340329:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763425.340334:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787763425.341038:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763425.341558:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763425.341808:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763425.342036:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787763425.980415:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763425.980421:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763425.980426:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763426.364273:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763426.364278:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763426.364280:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763426.364284:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763426.364287:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763426.364289:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763426.364291:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763426.364293:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763426.364295:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763427.004513:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763427.004520:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763427.004528:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763427.388357:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763427.388366:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763427.388370:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763427.388376:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763427.388381:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763427.388385:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763427.388388:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763427.388391:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763427.388395:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763428.028326:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763428.028332:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763428.028339:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763428.412445:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763428.412453:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763428.412456:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763428.412461:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763428.412465:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763428.412469:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763428.412472:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763428.412475:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763428.412478:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763429.052398:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763429.052403:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763429.052407:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763429.436294:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763429.436305:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763429.436309:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763429.436316:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763429.436320:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763429.436324:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763429.436328:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763429.436337:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763429.436340:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763430.076438:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763430.076444:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763430.076449:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763430.460343:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763430.460352:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763430.460359:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763430.460369:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763430.460379:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763430.460387:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763430.460393:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763430.460400:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763430.460407:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787763430.461471:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763430.461478:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787763430.462658:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763430.462661:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787763430.463101:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763430.463105:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787763430.463677:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763430.463682:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:0.0:1787763431.100424:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763431.100431:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763431.100436:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763431.484434:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763431.484441:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763431.484444:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763431.484451:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763431.484458:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763431.484463:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763431.484467:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763431.484471:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763431.484474:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763432.124430:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763432.124439:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763432.124446:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763432.508250:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763432.508260:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763432.508263:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763432.508270:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763432.508275:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763432.508281:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763432.508285:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763432.508288:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763432.508292:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763433.148420:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763433.148426:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763433.148430:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763433.532278:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763433.532289:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763433.532292:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763433.532300:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763433.532320:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763433.532325:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763433.532328:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763433.532332:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763433.532336:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763434.172282:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763434.172286:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763434.172289:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763434.556297:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763434.556306:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763434.556310:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763434.556317:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763434.556323:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763434.556328:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763434.556332:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763434.556335:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763434.556339:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763435.196367:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763435.196372:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763435.196375:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763435.580228:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763435.580238:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763435.580241:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763435.580248:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763435.580254:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763435.580261:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763435.580268:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763435.580274:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763435.580281:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787763435.581271:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763435.581281:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787763435.582223:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763435.582228:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787763435.582628:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763435.582632:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787763435.583009:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763435.583012:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:0.0:1787763436.220436:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763436.220443:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763436.220448:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763436.604266:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763436.604276:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763436.604279:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763436.604287:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763436.604293:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763436.604299:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763436.604303:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763436.604306:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763436.604310:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763437.628231:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763437.628240:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763437.628245:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763437.628255:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763437.628262:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763437.628269:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763437.628274:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763437.628279:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763437.628285:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763438.268478:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763438.268484:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763438.268489:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763438.652268:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763438.652278:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763438.652282:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763438.652289:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763438.652295:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763438.652301:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763438.652305:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763438.652311:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763438.652315:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763439.292318:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763439.292322:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763439.292326:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763439.676223:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763439.676230:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763439.676231:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763439.676236:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763439.676239:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763439.676242:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763439.676244:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763439.676246:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763439.676249:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763440.700244:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763440.700255:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763440.700259:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763440.700267:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763440.700273:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763440.700279:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763440.700283:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763440.700291:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763440.700299:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787763440.701074:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763440.701105:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787763440.702150:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763440.702154:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787763440.702390:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763440.702392:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787763440.702728:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763440.702731:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:0.0:1787763441.340315:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763441.340331:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763441.340333:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763441.724256:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763441.724268:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763441.724271:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763441.724279:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763441.724287:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763441.724292:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763441.724297:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763441.724301:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763441.724306:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763442.364448:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763442.364455:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763442.364460:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763442.748208:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763442.748215:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763442.748217:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763442.748222:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763442.748227:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763442.748231:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763442.748234:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763442.748236:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763442.748238:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763443.772266:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763443.772276:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763443.772279:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763443.772287:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763443.772293:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763443.772298:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763443.772302:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763443.772305:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763443.772310:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763444.412461:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763444.412467:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763444.412473:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763444.796261:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763444.796269:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763444.796271:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763444.796278:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763444.796295:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763444.796301:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763444.796305:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763444.796309:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763444.796313:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763445.436413:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763445.436419:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763445.436424:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763445.820238:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763445.820248:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763445.820251:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763445.820259:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763445.820266:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763445.820271:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763445.820275:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763445.820279:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763445.820284:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787763445.821332:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763445.821339:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787763445.822086:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763445.822090:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787763445.822547:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763445.822551:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787763445.822889:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763445.822893:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:0.0:1787763446.844266:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763446.844278:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763446.844281:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763446.844290:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763446.844296:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763446.844303:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763446.844307:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763446.844312:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763446.844318:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763447.484385:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763447.484392:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763447.484397:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763447.868263:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763447.868273:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763447.868277:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763447.868284:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763447.868291:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763447.868296:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763447.868300:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763447.868303:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763447.868307:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763448.508413:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763448.508420:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763448.508425:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763448.892248:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763448.892256:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763448.892259:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763448.892266:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763448.892272:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763448.892276:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763448.892280:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763448.892284:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763448.892289:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763449.916302:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763449.916319:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763449.916323:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763449.916332:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763449.916338:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763449.916346:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763449.916350:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763449.916355:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763449.916359:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763450.556286:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763450.556292:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763450.556295:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763450.940302:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763450.940315:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763450.940322:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763450.940336:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763450.940348:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763450.940357:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763450.940366:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763450.940371:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763450.940377:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787763450.941615:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763450.941634:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787763450.942732:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763450.942738:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787763450.943312:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763450.943316:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787763450.943705:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763450.943710:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000020:00000001:0.0:1787763451.580405:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763451.580412:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763451.580417:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763451.964310:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763451.964320:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763451.964323:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763451.964330:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763451.964335:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763451.964339:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763451.964342:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763451.964346:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763451.964349:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763453.628345:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763453.628352:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763453.628358:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763454.652424:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763454.652431:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763454.652436:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763456.060355:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763456.060365:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763456.060370:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763456.060376:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763456.060381:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763456.060391:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21170
00000400:00000200:0.0:1787763456.060402:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763456.060410:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763456.060414:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763456.060419:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763456.060423:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763456.060426:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763456.060429:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763456.060434:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763456.060437:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763456.060441:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763456.060446:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763456.060449:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763456.060451:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763456.060455:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763456.060458:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763456.060460:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763456.060463:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763456.060467:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763456.060472:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763456.060475:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763456.060478:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763456.060483:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763456.060487:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763456.060490:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763456.060493:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763456.060503:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763456.060508:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763456.060513:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763456.060516:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763456.060520:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763456.060523:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763456.060529:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763456.060538:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763456.060546:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763456.060558:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832626560/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763456.060572:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763456.060578:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763456.060583:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832626560/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763456.060615:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763456.060620:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763456.060624:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763456.060626:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763456.060628:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763456.060632:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763456.060636:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21170
00000400:00000200:0.0:1787763456.060640:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763456.060645:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763456.060648:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763456.060653:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763456.060657:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763456.060660:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763456.060664:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763456.060669:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763456.060674:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763456.060677:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763456.060685:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763456.060693:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763456.060698:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763456.060706:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763456.060717:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763456.060725:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763456.060732:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763456.060743:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763456.060746:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763456.060749:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763456.060753:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763456.060763:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763456.060775:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763456.060784:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763456.060796:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763456.060813:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763456.060824:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763456.060839:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763456.060849:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763456.060861:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763456.060868:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763456.060878:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763456.060887:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763456.060896:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763456.060907:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763456.060919:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763456.060939:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832626688/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763456.061025:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763456.061034:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763456.061054:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832626688/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763456.061077:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763456.061084:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763456.061093:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763456.061099:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763456.061103:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763456.061111:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763456.061121:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21170
00000400:00000200:0.0:1787763456.061193:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763456.061209:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763456.061219:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763456.061233:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763456.061240:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763456.061245:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763456.061252:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763456.061258:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763456.061262:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763456.061271:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763456.061277:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763456.061288:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:2.1:1787763456.061297:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787763456.061298:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:2.1:1787763456.061310:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000100:00000040:0.0:1787763456.061313:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763456.061321:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763456.061332:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763456.061339:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763456.061346:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763456.061359:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763456.061366:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763456.061373:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763456.061381:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763456.061393:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763456.061397:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763456.061405:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763456.061411:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763456.061419:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763456.061430:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763456.061437:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763456.061447:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763456.061451:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763456.061461:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763456.061466:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763456.061471:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763456.061480:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763456.061492:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763456.061504:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832626816/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763456.061530:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763456.061539:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763456.061548:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832626816/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763456.061565:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763456.061576:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763456.061590:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763456.061600:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763456.061611:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763456.061620:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763456.061623:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763456.061630:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763456.061639:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21170
00000400:00000200:0.0:1787763456.061647:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763456.061656:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763456.061664:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763456.061676:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763456.061683:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763456.061690:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763456.061697:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763456.061703:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763456.061708:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763456.061716:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763456.061720:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763456.061730:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763456.061734:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763456.061749:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763456.061758:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763456.061773:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763456.061787:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763456.061799:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763456.061817:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763456.061832:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763456.061845:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763456.061856:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763456.061867:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763456.061873:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763456.061882:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763456.061888:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763456.061898:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763456.061909:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763456.061916:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763456.061924:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763456.061931:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763456.061941:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763456.061946:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763456.061952:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763456.061960:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787763456.061971:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763456.062011:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832626944/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763456.062054:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763456.062062:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763456.062075:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832626944/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763456.062101:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763456.062115:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21180)
00000800:00000200:2.1:1787763456.062398:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763456.062402:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787763456.062826:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763456.062829:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787763456.063250:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763456.063253:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000020:00000001:0.0:1787763456.700397:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763456.700403:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763456.700408:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763457.724370:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763457.724378:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763457.724384:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763459.772405:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763459.772411:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763459.772416:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763460.796258:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763460.796262:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763460.796264:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763461.180277:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763461.180284:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763461.180286:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763461.180289:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763461.180293:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763461.180300:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21175
00000400:00000200:0.0:1787763461.180306:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763461.180357:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763461.180360:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763461.180362:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763461.180365:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763461.180367:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763461.180368:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763461.180372:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763461.180374:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763461.180376:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763461.180379:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763461.180381:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763461.180382:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763461.180384:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763461.180386:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763461.180387:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763461.180388:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763461.180391:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763461.180393:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763461.180395:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763461.180397:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763461.180399:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763461.180401:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763461.180403:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763461.180404:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763461.180413:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000020:00000040:0.0:1787763461.180415:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763461.180418:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763461.180419:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763461.180421:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763461.180424:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763461.180427:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763461.180434:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763461.180439:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763461.180446:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832627072/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763461.180456:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763461.180462:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763461.180468:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832627072/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763461.180490:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763461.180494:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763461.180497:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763461.180499:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763461.180502:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763461.180505:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763461.180509:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21175
00000400:00000200:0.0:1787763461.180513:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763461.180516:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763461.180520:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763461.180524:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763461.180528:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763461.180534:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763461.180536:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763461.180540:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763461.180550:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763461.180553:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763461.180557:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763461.180561:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763461.180564:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763461.180570:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763461.180573:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763461.180577:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763461.180580:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763461.180584:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763461.180589:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763461.180593:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763461.180596:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763461.180603:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763461.180608:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763461.180613:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763461.180617:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763461.180622:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763461.180628:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763461.180636:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000020:00000040:0.0:1787763461.180643:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763461.180650:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763461.180654:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763461.180657:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763461.180660:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763461.180663:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763461.180669:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787763461.180682:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763461.180690:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832627200/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763461.180704:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763461.180709:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763461.180715:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832627200/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763461.180727:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763461.180732:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763461.180738:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763461.180741:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763461.180745:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763461.180748:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763461.180752:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21175
00000400:00000200:0.0:1787763461.180761:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763461.180769:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763461.180774:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000200:2.1:1787763461.180779:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763461.180780:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763461.180784:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.1:1787763461.180784:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000100:00000040:0.0:1787763461.180790:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763461.180796:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763461.180800:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763461.180801:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763461.180803:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763461.180806:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763461.180811:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763461.180816:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763461.180826:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763461.180832:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763461.180839:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763461.180845:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763461.180850:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763461.180856:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763461.180860:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763461.180863:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763461.180867:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763461.180871:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763461.180875:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763461.180878:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763461.180883:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763461.180886:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763461.180894:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
00000020:00000040:0.0:1787763461.180897:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763461.180900:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763461.180901:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763461.180903:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763461.180906:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763461.180909:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763461.180915:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787763461.180919:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763461.180926:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832627328/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763461.180934:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763461.180937:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763461.180955:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832627328/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763461.180968:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763461.180971:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763461.180976:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763461.180980:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763461.180986:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763461.180989:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763461.180992:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763461.180997:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763461.181002:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21175
00000400:00000200:0.0:1787763461.181008:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763461.181014:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763461.181017:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763461.181022:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763461.181026:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763461.181028:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763461.181029:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763461.181032:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763461.181036:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763461.181040:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763461.181045:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763461.181051:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763461.181057:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763461.181067:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763461.181074:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763461.181082:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763461.181088:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763461.181094:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763461.181099:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763461.181104:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763461.181108:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763461.181112:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763461.181115:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763461.181120:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763461.181125:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763461.181156:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763461.181161:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763461.181167:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763461.181173:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763461.181177:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763461.181180:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763461.181184:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763461.181186:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763461.181189:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763461.181193:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787763461.181198:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763461.181205:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832627456/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763461.181215:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763461.181219:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763461.181227:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832627456/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763461.181250:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763461.181256:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21185)
00000800:00000200:2.1:1787763461.181375:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763461.181377:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787763461.181605:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763461.181607:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787763461.181789:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763461.181791:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000020:00000001:0.0:1787763462.845684:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763462.845691:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763462.845697:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763463.868295:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763463.868301:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763463.868310:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763465.916260:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763465.916264:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763465.916267:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763466.300300:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763466.300310:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763466.300314:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763466.300319:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763466.300325:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763466.300361:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21180
00000400:00000200:0.0:1787763466.300385:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763466.300393:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763466.300397:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763466.300402:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763466.300406:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763466.300410:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763466.300413:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763466.300418:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763466.300422:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763466.300425:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763466.300429:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763466.300433:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763466.300436:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763466.300441:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763466.300445:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763466.300448:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763466.300451:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763466.300455:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763466.300459:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763466.300463:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763466.300466:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763466.300490:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763466.300495:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763466.300498:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763466.300502:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763466.300512:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763466.300517:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763466.300522:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763466.300525:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763466.300529:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763466.300534:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763466.300538:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763466.300548:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787763466.300557:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763466.300576:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832627584/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763466.300589:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763466.300595:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763466.300602:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832627584/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763466.300630:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763466.300634:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763466.300638:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763466.300640:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763466.300642:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763466.300664:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763466.300670:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21180
00000400:00000200:0.0:1787763466.300676:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763466.300684:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763466.300691:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763466.300698:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763466.300701:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763466.300704:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763466.300708:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763466.300715:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763466.300718:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763466.300722:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763466.300726:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763466.300734:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763466.300743:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763466.300754:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763466.300762:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763466.300769:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763466.300777:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763466.300786:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763466.300804:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763466.300812:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763466.300818:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763466.300828:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763466.300835:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763466.300841:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763466.300850:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763466.300859:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763466.300865:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763466.300872:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763466.300879:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763466.300889:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763466.300894:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763466.300900:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763466.300924:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763466.300929:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763466.300941:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763466.300950:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763466.300965:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832627712/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763466.300983:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763466.300990:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763466.301000:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832627712/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763466.301016:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763466.301022:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763466.301031:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763466.301036:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763466.301040:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763466.301048:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763466.301059:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21180
00000400:00000200:0.0:1787763466.301072:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763466.301085:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763466.301092:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763466.301103:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763466.301111:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763466.301120:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763466.301123:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763466.301125:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763466.301129:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763466.301159:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763466.301162:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763466.301167:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763466.301170:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763466.301176:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763466.301182:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763466.301188:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763466.301192:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763466.301196:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763466.301203:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763466.301206:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763466.301226:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763466.301234:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763466.301244:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763466.301247:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763466.301254:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000800:00000200:2.1:1787763466.301262:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000008:00000001:0.0:1787763466.301263:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763466.301271:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787763466.301272:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000100:00000010:0.0:1787763466.301286:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763466.301300:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763466.301312:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763466.301321:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763466.301346:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763466.301352:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763466.301358:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763466.301370:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787763466.301380:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763466.301395:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832627840/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763466.301415:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763466.301421:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763466.301430:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832627840/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763466.301445:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763466.301455:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763466.301466:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763466.301475:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763466.301484:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763466.301487:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763466.301491:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763466.301498:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763466.301508:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21180
00000400:00000200:0.0:1787763466.301518:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763466.301521:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763466.301524:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763466.301528:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763466.301531:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763466.301536:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763466.301540:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763466.301545:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763466.301547:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763466.301552:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763466.301556:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763466.301561:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763466.301569:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763466.301582:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763466.301589:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763466.301599:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763466.301604:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763466.301608:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763466.301613:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763466.301617:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763466.301620:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763466.301627:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763466.301635:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763466.301642:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763466.301648:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763466.301655:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763466.301662:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763466.301670:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763466.301676:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763466.301683:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763466.301687:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763466.301700:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763466.301703:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763466.301707:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763466.301716:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787763466.301724:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763466.301755:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832627968/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763466.301784:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763466.301792:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763466.301803:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832627968/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763466.301829:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763466.301841:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21191)
00000800:00000200:2.1:1787763466.302223:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763466.302226:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787763466.302617:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763466.302619:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787763466.302939:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763466.302941:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:0.0:1787763466.940409:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763466.940415:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763466.940420:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763467.324431:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763467.324442:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763467.324447:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763467.324456:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763467.324464:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763467.324470:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763467.324475:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763467.324482:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763467.324487:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763471.036378:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763471.036387:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763471.036393:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:2.1:1787763471.420891:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763471.420898:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787763471.421470:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763471.421471:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787763471.421677:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763471.421679:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787763471.421884:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763471.421886:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:0.0:1787763474.108401:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763474.108407:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763474.108412:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787763476.541347:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763476.541356:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787763476.542199:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763476.542203:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787763476.542570:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763476.542573:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787763476.542939:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763476.542942:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:0.0:1787763477.180336:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763477.180340:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763477.180344:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763478.204307:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763478.204312:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763478.204322:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763479.228420:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763479.228424:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763479.228428:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763480.252389:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763480.252396:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763480.252401:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763481.276342:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763481.276349:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763481.276353:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.1:1787763481.661514:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763481.661559:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787763481.662508:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763481.662512:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787763481.662955:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763481.662961:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787763481.663385:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763481.663388:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:0.0:1787763482.300443:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763482.300450:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763482.300455:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763483.324399:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763483.324408:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763483.324415:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763484.348399:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763484.348405:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763484.348409:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763485.372347:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763485.372351:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763485.372355:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763486.396350:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763486.396356:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763486.396360:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.1:1787763486.780960:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763486.780968:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787763486.781486:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763486.781488:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787763486.781916:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763486.781919:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787763486.782122:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763486.782124:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000020:00000001:0.0:1787763487.420384:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763487.420391:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763487.420396:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763488.444483:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763488.444489:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763488.444494:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763489.468478:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763489.468485:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763489.468491:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763490.492442:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763490.492449:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763490.492454:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763491.516434:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763491.516459:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763491.516466:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.1:1787763491.901236:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763491.901975:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763491.902446:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763491.902700:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787763492.540413:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763492.540417:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763492.540421:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763493.564393:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763493.564398:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763493.564402:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763494.588433:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763494.588437:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763494.588440:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763495.612441:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763495.612447:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763495.612452:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763496.636388:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763496.636392:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763496.636395:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:1.1:1787763497.021025:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763497.021037:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787763497.022088:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763497.022091:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787763497.022361:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763497.022363:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787763497.022572:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763497.022574:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:0.0:1787763497.660386:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763497.660391:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763497.660394:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763498.684406:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763498.684411:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763498.684414:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763499.068246:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763499.068254:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763499.068256:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763499.068260:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763499.068264:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763499.068267:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763499.068269:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763499.068272:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763499.068274:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763499.708341:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763499.708346:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763499.708349:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763500.092280:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763500.092290:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763500.092292:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763500.092299:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763500.092303:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763500.092307:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763500.092310:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763500.092314:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763500.092327:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763500.732302:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763500.732307:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763500.732311:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763501.116213:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763501.116222:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763501.116224:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763501.116230:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763501.116234:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763501.116237:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763501.116240:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763501.116243:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763501.116246:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763502.140255:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763502.140266:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763502.140268:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763502.140278:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763502.140307:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763502.140313:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763502.140317:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763502.140320:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763502.140323:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763502.141115:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763502.141121:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787763502.141821:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763502.141823:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787763502.142187:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763502.142190:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787763502.142559:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763502.142561:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:0.0:1787763502.780322:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763502.780326:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763502.780329:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763505.852310:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763505.852315:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763505.852318:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787763507.260777:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763507.260782:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787763507.261249:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763507.261251:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787763507.261440:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763507.261441:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787763507.261611:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763507.261613:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787763512.380856:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763512.380862:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787763512.381523:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763512.381526:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787763512.381864:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763512.381867:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787763512.382267:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763512.382270:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787763517.501166:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763517.501169:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787763517.501431:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763517.501433:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787763517.501677:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763517.501679:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787763517.501903:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763517.501906:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000100:00080000:0.0:1787763522.620253:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763522.620262:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763522.620265:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763522.620270:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763522.620275:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763522.620283:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21237
00000400:00000200:0.0:1787763522.620292:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763522.620298:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763522.620301:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763522.620304:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763522.620307:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763522.620309:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763522.620311:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763522.620314:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763522.620316:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763522.620318:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763522.620322:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763522.620323:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763522.620326:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763522.620328:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763522.620330:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763522.620331:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763522.620334:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763522.620336:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763522.620339:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763522.620341:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763522.620344:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763522.620347:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763522.620350:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763522.620352:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763522.620354:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763522.620364:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000020:00000040:0.0:1787763522.620368:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763522.620373:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763522.620375:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763522.620379:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763522.620382:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763522.620387:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763522.620398:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763522.620408:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763522.620419:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832633216/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763522.620430:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763522.620434:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763522.620438:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832633216/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763522.620456:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763522.620459:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763522.620462:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763522.620463:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763522.620464:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763522.620467:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763522.620470:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21237
00000400:00000200:0.0:1787763522.620475:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763522.620478:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763522.620483:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763522.620486:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763522.620489:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763522.620492:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763522.620496:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763522.620498:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763522.620501:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763522.620504:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763522.620507:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763522.620533:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763522.620540:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763522.620548:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763522.620554:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763522.620559:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763522.620563:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763522.620568:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763522.620572:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763522.620577:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763522.620582:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763522.620591:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763522.620598:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763522.620601:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763522.620609:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763522.620615:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763522.620617:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763522.620624:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787763522.620626:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763522.620629:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763522.620632:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763522.620635:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763522.620638:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763522.620640:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763522.620645:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763522.620652:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763522.620659:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832633344/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763522.620671:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763522.620676:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763522.620681:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832633344/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763522.620704:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763522.620710:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763522.620718:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763522.620722:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763522.620728:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763522.620734:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763522.620742:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21237
00000400:00000200:0.0:1787763522.620749:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763522.620757:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763522.620759:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763522.620761:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763522.620763:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763522.620765:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763522.620766:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763522.620769:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763522.620771:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763522.620773:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763522.620775:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763522.620780:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763522.620785:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763522.620788:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763522.620792:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763522.620794:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763522.620795:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763522.620799:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763522.620803:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763522.620806:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763522.620808:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763522.620811:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763522.620818:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763522.620820:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:3.1:1787763522.620823:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000008:00100000:0.0:1787763522.620824:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763522.620829:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763522.620831:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:0.0:1787763522.620832:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763522.620838:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
00000020:00000040:0.0:1787763522.620844:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763522.620848:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763522.620851:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763522.620852:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763522.620853:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763522.620855:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763522.620862:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763522.620867:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763522.620874:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832633472/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763522.620885:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763522.620887:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763522.620891:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832633472/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763522.620900:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763522.620905:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763522.620912:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763522.620918:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763522.620921:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763522.620923:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763522.620924:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763522.620929:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763522.620933:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21237
00000400:00000200:0.0:1787763522.620940:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763522.620945:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763522.620949:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763522.620955:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763522.620958:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763522.620962:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763522.620965:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763522.620970:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763522.620971:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763522.620972:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763522.620974:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763522.620976:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763522.620977:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763522.620979:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763522.620984:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763522.620987:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763522.620991:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763522.620996:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763522.620999:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763522.621003:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763522.621006:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763522.621009:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763522.621011:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763522.621015:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763522.621018:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763522.621020:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763522.621022:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763522.621028:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763522.621032:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763522.621034:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763522.621037:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763522.621040:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763522.621043:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763522.621046:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763522.621051:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787763522.621057:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763522.621064:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832633600/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763522.621092:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763522.621098:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763522.621106:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832633600/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763522.621118:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763522.621123:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21247)
00000800:00000200:3.1:1787763522.621354:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763522.621356:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787763522.621601:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763522.621603:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787763522.621799:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763522.621800:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787763527.740854:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763527.740858:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787763527.741511:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763527.741513:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787763527.741719:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763527.741721:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787763527.741955:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763527.741957:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:0.0:1787763528.764226:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763528.764234:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763528.764236:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763528.764242:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763528.764247:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763528.764251:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763528.764254:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763528.764258:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763528.764262:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763529.788230:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763529.788240:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763529.788242:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763529.788248:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763529.788251:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763529.788254:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763529.788256:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763529.788259:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763529.788261:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763530.812212:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763530.812222:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763530.812225:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763530.812231:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763530.812236:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763530.812240:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763530.812243:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763530.812246:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763530.812250:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763531.836221:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763531.836228:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763531.836231:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763531.836238:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763531.836243:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763531.836248:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763531.836251:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763531.836255:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763531.836259:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763532.860257:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763532.860267:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763532.860273:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763532.860284:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763532.860293:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763532.860298:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763532.860307:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763532.860314:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763532.860323:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787763532.861673:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763532.861679:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787763532.862190:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763532.862194:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787763532.862500:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763532.862502:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787763532.862740:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763532.862742:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:0.0:1787763533.884320:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763533.884331:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763533.884335:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763533.884344:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763533.884351:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763533.884357:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763533.884362:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763533.884367:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763533.884373:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763534.908243:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763534.908255:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763534.908259:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763534.908281:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763534.908288:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763534.908293:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763534.908297:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763534.908302:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763534.908308:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763535.932232:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763535.932243:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763535.932246:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763535.932253:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763535.932258:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763535.932263:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763535.932266:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763535.932270:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763535.932274:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763536.956234:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763536.956244:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763536.956247:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763536.956253:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763536.956257:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763536.956261:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763536.956264:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763536.956268:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763536.956272:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787763537.981168:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763537.981171:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787763537.982279:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763537.982292:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787763537.982543:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763537.982544:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787763537.982730:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763537.982731:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787763543.101068:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763543.101077:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787763543.101516:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763543.101518:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787763543.101733:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763543.101736:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787763543.101928:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763543.101930:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:0.0:1787763546.172316:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763546.172326:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763546.172329:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763546.172337:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763546.172341:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763546.172347:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763546.172351:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763546.172368:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763546.172372:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763547.196309:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763547.196321:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763547.196325:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763547.196335:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763547.196343:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763547.196373:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763547.196378:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763547.196383:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763547.196389:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787763548.220876:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763548.220882:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787763548.221572:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763548.221574:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787763548.221836:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763548.221838:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787763548.222052:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763548.222054:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787763553.340751:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763553.340758:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1787763553.341299:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763553.341301:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1787763553.341559:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763553.341562:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1787763553.341776:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763553.341778:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1787763558.460970:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763558.461655:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763558.462022:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763558.462232:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787763558.462297:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763558.462300:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763559.100308:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763559.100319:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763559.100323:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763559.100331:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763559.100338:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763559.100343:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763559.100347:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763559.100351:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763559.100357:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763560.124313:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763560.124328:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763560.124336:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763560.124353:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763560.124364:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763560.124374:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763560.124385:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763560.124394:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763560.124405:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763560.124531:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763560.124538:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763560.124543:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763561.148339:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763561.148349:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763561.148354:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763561.148364:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763561.148372:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763561.148378:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763561.148387:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763561.148391:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763561.148395:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:0.0:1787763561.532385:0:257563:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.110:988
00000400:00000100:0.0:1787763561.532427:0:257563:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.110@tcp at host 192.168.203.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:0.0:1787763561.532438:0:257563:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763561.532442:0:257563:0:(router.c:1791:lnet_notify()) 192.168.203.10@tcp notifying 192.168.203.110@tcp: down
00000400:00000200:0.0:1787763561.532447:0:257563:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763561.532451:0:257563:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:0.0:1787763561.532456:0:257563:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:0.0:1787763561.532460:0:257563:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1000.
00000400:00000200:0.0:1787763561.532466:0:257563:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1787763561.532470:0:257563:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1787763561.532476:0:257563:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933000
00000400:00000010:0.0:1787763561.532480:0:257563:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933000.
00000100:00000001:0.0:1787763561.532484:0:257563:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763561.532494:0:257563:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f87bde00 x1874599832636800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763583 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763561.532516:0:257563:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763561.532521:0:257563:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bde00 x1874599832636800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763583 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763561.532535:0:257563:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763561.532537:0:257563:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:0.0:1787763561.532549:0:257563:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000100:00000001:1.0:1787763561.532550:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000800:00000010:0.0:1787763561.532552:0:257563:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1200.
00000400:00000200:0.0:1787763561.532557:0:257563:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00000001:1.0:1787763561.532557:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763561.532563:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000400:00000200:0.0:1787763561.532564:0:257563:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1787763561.532570:0:257563:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933088
00000400:00000010:0.0:1787763561.532572:0:257563:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933088.
00000100:00000001:0.0:1787763561.532577:0:257563:0:(events.c:34:request_out_callback()) Process entered
00000100:00100000:1.0:1787763561.532577:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763558/real 1787763561]  req@ffff9751f87bde00 x1874599832636800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763561.532585:0:257563:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f87bc780 x1874599832636928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763583 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763561.532601:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:0.0:1787763561.532602:0:257563:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000400:00000001:1.0:1787763561.532606:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:0.0:1787763561.532608:0:257563:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc780 x1874599832636928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763583 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1787763561.532611:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210e70.
00000100:00000001:0.0:1787763561.532620:0:257563:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763561.532622:0:257563:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:0.0:1787763561.532627:0:257563:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:0.0:1787763561.532631:0:257563:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1a00.
00000400:00000200:1.0:1787763561.532633:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933d48
00000400:00000200:0.0:1787763561.532635:0:257563:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1787763561.532641:0:257563:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000010:1.0:1787763561.532643:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933d48.
00000400:00000200:0.0:1787763561.532644:0:257563:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933cc0
00000400:00000010:0.0:1787763561.532649:0:257563:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933cc0.
00000100:00000001:1.0:1787763561.532652:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:0.0:1787763561.532654:0:257563:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763561.532661:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bde00 x1874599832636800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787763561.532663:0:257563:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f87bc3c0 x1874599832637056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763583 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763561.532690:0:257563:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000200:1.0:1787763561.532693:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bde00 x1874599832636800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763561.532697:0:257563:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc3c0 x1874599832637056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763583 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763561.532716:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763561.532718:0:257563:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763561.532720:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763561.532721:0:257563:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763561.532726:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000800:00000100:0.0:1787763561.532731:0:257563:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000100:00000001:1.0:1787763561.532732:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000800:00000010:0.0:1787763561.532738:0:257563:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:0.0:1787763561.532742:0:257563:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00100000:1.0:1787763561.532743:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bde00 x1874599832636800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787763561.532752:0:257563:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000100:00000001:1.0:1787763561.532758:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1787763561.532760:0:257563:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933880
00000400:00000010:0.0:1787763561.532764:0:257563:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933880.
00000100:00000040:1.0:1787763561.532768:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bde00 x1874599832636800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763561.532769:0:257563:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763561.532777:0:257563:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f87bfc00 x1874599832637184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763583 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763561.532788:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763561.532794:0:257563:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787763561.532795:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763561.532801:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:0.0:1787763561.532802:0:257563:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bfc00 x1874599832637184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763583 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763561.532803:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763561.532809:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763561.532813:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763561.532815:0:257563:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763561.532819:0:257563:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:1.0:1787763561.532820:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763561.532829:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763561.532832:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763561.532839:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bde00 x1874599832636800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763561.532849:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832636800:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763561.532854:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763561.532857:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763561.532861:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bde00 x1874599832636800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763561.532869:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763561.532872:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763561.532876:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787763561.532881:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763561.532885:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763561.532886:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763561.532889:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763561.532892:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610c00.
02000000:00000001:1.0:1787763561.532895:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763561.532897:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763561.532899:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bde00.
00000100:00000001:1.0:1787763561.532903:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763561.532904:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763561.532907:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763561.532909:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763561.532911:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763561.532912:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763561.532916:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763558/real 1787763561]  req@ffff9751f87bfc00 x1874599832637184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763561.532925:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763561.532927:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763561.532929:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210840.
00000400:00000200:1.0:1787763561.532932:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933990
00000400:00000010:1.0:1787763561.532934:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933990.
00000100:00000001:1.0:1787763561.532937:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763561.532940:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bfc00 x1874599832637184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763561.532950:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bfc00 x1874599832637184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763561.532957:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763561.532959:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763561.532961:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763561.532963:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763561.532967:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bfc00 x1874599832637184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763561.532975:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763561.532979:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bfc00 x1874599832637184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763561.532987:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763561.532990:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763561.532991:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763561.532992:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763561.532994:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763561.532996:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763561.532999:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763561.533003:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763561.533005:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763561.533010:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bfc00 x1874599832637184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763561.533018:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832637184:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763561.533022:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763561.533026:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763558/real 1787763561]  req@ffff9751f87bc3c0 x1874599832637056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763561.533035:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763561.533037:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763561.533039:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92101b8.
00000400:00000200:1.0:1787763561.533042:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763561.533044:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763561.533046:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763561.533050:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc3c0 x1874599832637056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763561.533057:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc3c0 x1874599832637056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763561.533062:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763561.533063:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763561.533065:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763561.533066:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763561.533069:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc3c0 x1874599832637056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763561.533073:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763561.533076:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc3c0 x1874599832637056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763561.533080:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763561.533082:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763561.533083:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763561.533084:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763561.533086:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763561.533087:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763561.533089:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763561.533091:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763561.533092:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763561.533095:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc3c0 x1874599832637056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763561.533099:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832637056:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763561.533102:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763561.533105:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763558/real 1787763561]  req@ffff9751f87bc780 x1874599832636928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763561.533112:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763561.533113:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763561.533116:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763561.533119:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763561.533121:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763561.533123:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763561.533126:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc780 x1874599832636928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763561.533157:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc780 x1874599832636928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763561.533166:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763561.533168:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763561.533170:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763561.533172:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763561.533176:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc780 x1874599832636928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763561.533182:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763561.533186:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc780 x1874599832636928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763561.533191:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763561.533193:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763561.533194:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763561.533195:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763561.533196:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763561.533207:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763561.533209:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763561.533212:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763561.533214:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763561.533217:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc780 x1874599832636928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763561.533224:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832636928:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763561.533229:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763561.533231:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763561.533234:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bfc00 x1874599832637184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763561.533241:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763561.533243:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763561.533245:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763561.533248:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763561.533264:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763561.533266:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763561.533268:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763561.533270:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612000.
02000000:00000001:1.0:1787763561.533273:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763561.533275:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763561.533277:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bfc00.
00000100:00000001:1.0:1787763561.533280:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763561.533281:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763561.533283:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763561.533287:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc3c0 x1874599832637056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763561.533293:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763561.533295:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763561.533296:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763561.533299:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763561.533301:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763561.533302:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763561.533304:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763561.533306:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610800.
02000000:00000001:1.0:1787763561.533308:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763561.533310:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763561.533311:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc3c0.
00000100:00000001:1.0:1787763561.533314:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763561.533315:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763561.533316:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763561.533320:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc780 x1874599832636928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763583 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763561.533326:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763561.533327:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763561.533329:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763561.533331:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763561.533332:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763561.533333:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763561.533335:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763561.533336:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611c00.
02000000:00000001:1.0:1787763561.533338:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763561.533339:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763561.533340:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc780.
00000100:00000001:1.0:1787763561.533343:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763561.533344:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763561.533346:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763561.533353:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763561.533354:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763561.533356:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763561.533358:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763562.172266:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763562.172273:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763562.172275:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763562.172282:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763562.172304:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763562.172307:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763562.172312:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763562.172316:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763562.172319:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763563.196230:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763563.196236:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763563.196253:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763563.196263:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763563.196269:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763563.196274:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763563.196277:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763563.196281:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763563.196285:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763563.196340:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763563.196343:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763563.196346:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763563.580269:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763563.580275:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763563.580278:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763563.580280:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763563.580284:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763563.580290:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21278
00000400:00000200:0.0:1787763563.580297:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763563.580302:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763563.580304:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763563.580307:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763563.580309:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763563.580311:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763563.580312:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763563.580315:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763563.580318:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763563.580320:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763563.580323:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763563.580325:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763563.580327:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763563.580329:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763563.580330:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763563.580331:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763563.580333:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763563.580335:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763563.580337:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763563.580338:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763563.580341:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763563.580344:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763563.580346:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763563.580347:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763563.580349:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763563.580354:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763563.580356:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763563.580359:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763563.580360:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763563.580363:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763563.580365:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763563.580368:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763563.580374:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787763563.580379:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763563.580386:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832637312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763563.580394:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763563.580398:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763563.580401:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832637312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763563.580428:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763563.580432:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763563.580435:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763563.580437:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763563.580440:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763563.580443:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763563.580446:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21278
00000400:00000200:0.0:1787763563.580450:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763563.580454:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763563.580456:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763563.580465:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763563.580467:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763563.580469:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763563.580471:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763563.580472:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763563.580477:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:1.0:1787763563.580479:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000040:0.0:1787763563.580480:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763563.580484:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:1.0:1787763563.580484:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1787763563.580490:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763563.580493:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787763563.580496:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832637312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787763563.580497:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763563.580505:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763563.580512:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763563.580516:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763563.580519:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763563.580521:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787763563.580523:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787763563.580526:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:1.0:1787763563.580530:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:0.0:1787763563.580532:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:1.0:1787763563.580535:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763563.580536:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763563.580542:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763563.580543:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763563.580550:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:1.0:1787763563.580551:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832637312:192.168.203.110@tcp:250:kworker.0
02000000:00000001:0.0:1787763563.580556:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763563.580560:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1787763563.580561:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1787763563.580567:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:0.0:1787763563.580569:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763563.580574:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763563.580576:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763563.580581:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763563.580584:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763563.580584:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763563.580592:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1787763563.580593:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763563.580597:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763563.580599:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763563.580601:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763563.580604:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763563.580604:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763563.580611:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763563.580614:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000400:00000200:1.0:1787763563.580620:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763563.580622:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763563.580626:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000100:00100000:0.0:1787763563.580632:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832637440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1787763563.580637:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000001:0.0:1787763563.580645:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:1.0:1787763563.580648:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832637312, portal 25
00000100:00000001:0.0:1787763563.580649:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763563.580656:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000040:0.0:1787763563.580658:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832637440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.580660:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00000040:1.0:1787763563.580673:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832637312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763588 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763563.580675:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763563.580682:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763563.580691:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763563.580696:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1787763563.580696:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:0.0:1787763563.580701:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:1.0:1787763563.580705:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763563.580707:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:1.0:1787763563.580710:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00080000:0.0:1787763563.580714:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21278
00000100:00000200:1.0:1787763563.580718:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832637312, offset 0
00000400:00000200:0.0:1787763563.580724:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763563.580733:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:1.0:1787763563.580734:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00080000:0.0:1787763563.580737:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763563.580743:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763563.580746:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787763563.580747:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:0.0:1787763563.580753:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763563.580757:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:1.0:1787763563.580760:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000020:00000001:0.0:1787763563.580762:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763563.580765:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763563.580768:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:1.0:1787763563.580769:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:0.0:1787763563.580772:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763563.580777:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:1.0:1787763563.580777:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000020:00000001:0.0:1787763563.580782:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:1.0:1787763563.580787:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000040:0.0:1787763563.580790:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763563.580795:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787763563.580795:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000040:0.0:1787763563.580800:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763563.580806:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:1.0:1787763563.580807:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58214:58214:256:4294967295] 192.168.203.110@tcp LPNI seq info [58214:58214:8:4294967295]
00000020:00000040:0.0:1787763563.580811:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:1.0:1787763563.580819:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:0.0:1787763563.580820:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763563.580826:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763563.580834:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787763563.580837:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
02000000:00000001:0.0:1787763563.580840:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763563.580845:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763563.580849:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:1.0:1787763563.580850:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000008:00100000:0.0:1787763563.580853:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763563.580857:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1787763563.580859:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000100:00000001:0.0:1787763563.580860:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763563.580869:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000800:00000200:1.0:1787763563.580873:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:0.0:1787763563.580875:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763563.580881:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:1.0:1787763563.580882:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
02000000:00000001:0.0:1787763563.580883:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763563.580886:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763563.580890:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763563.580893:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787763563.580894:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763563.580899:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1787763563.580904:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763563.580907:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
02000000:00000010:0.0:1787763563.580914:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000400:00000200:1.0:1787763563.580916:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1787763563.580921:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763563.580927:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00100000:0.0:1787763563.580932:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832637568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787763563.580936:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763563.580942:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:0.0:1787763563.580946:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763563.580950:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1787763563.580951:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763563.580961:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832637568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763563.580961:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e580 x1874599832637312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763588 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.580989:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1787763563.580990:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763563.580995:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787763563.580996:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832637312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763588 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.581004:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763563.581009:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:0.0:1787763563.581010:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763563.581016:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787763563.581016:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763563.581025:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000200:1.0:1787763563.581025:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763563.581030:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1787763563.581031:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315951859
00000100:00080000:0.0:1787763563.581035:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000010:1.0:1787763563.581038:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4c00.
00000100:00000001:0.0:1787763563.581041:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1787763563.581047:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00080000:0.0:1787763563.581048:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21278
00000400:00000200:0.0:1787763563.581056:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763563.581058:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315951859
00000400:00000200:0.0:1787763563.581062:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000010:1.0:1787763563.581066:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c43c0.
00000100:00080000:0.0:1787763563.581067:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787763563.581073:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000040:0.0:1787763563.581074:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763563.581079:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787763563.581081:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1787763563.581083:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763563.581087:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763563.581088:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787763563.581092:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000020:00000001:0.0:1787763563.581093:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763563.581097:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763563.581100:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00100000:1.0:1787763563.581102:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763563/real 1787763563]  req@ffff9751f876e580 x1874599832637312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787763563.581104:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763563.581110:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763563.581116:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000001:1.0:1787763563.581125:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:0.0:1787763563.581126:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000001:1.0:1787763563.581130:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:0.0:1787763563.581131:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:1.0:1787763563.581160:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763563.581164:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763563.581166:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763563.581169:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763563.581174:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832637312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763563.581181:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763563.581186:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763563.581193:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000200:1.0:1787763563.581194:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832637312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787763563.581202:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763563.581207:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763563.581209:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763563.581211:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787763563.581213:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787763563.581217:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:0.0:1787763563.581218:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763563.581221:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787763563.581223:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763563.581228:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:1.0:1787763563.581231:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832637312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1787763563.581232:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763563.581253:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763563.581256:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763563.581261:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000100:00000001:1.0:1787763563.581263:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:0.0:1787763563.581265:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763563.581271:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763563.581273:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763563.581274:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832637312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763563.581276:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763563.581279:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763563.581283:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763563.581290:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787763563.581297:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763563.581305:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00100000:0.0:1787763563.581308:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832637696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763563.581311:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763563.581318:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763563.581321:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763563.581323:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763563.581327:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787763563.581329:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763563.581332:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:0.0:1787763563.581337:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832637696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787763563.581340:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763563.581351:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763563.581357:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763563.581358:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763563.581366:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21288)
00000100:00000040:1.0:1787763563.581369:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832637312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763563.581382:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832637312:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763563.581388:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763563.581396:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763563.581400:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832637312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.581407:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763563.581409:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763563.581412:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763563.581416:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763563.581417:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763563.581419:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763563.581421:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763563.581424:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:1.0:1787763563.581428:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763563.581429:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763563.581431:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:1.0:1787763563.581436:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763563.581437:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763563.581439:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763563.581447:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763563.581449:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763563.581452:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763563.581454:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763563.581455:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763563.581460:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832637440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.581467:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763563.581469:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763563.581472:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763563.581474:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763563.581476:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763563.581479:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832637440:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763563.581483:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763563.581485:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763563.581487:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763563.581489:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763563.581492:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763563.581495:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763563.581498:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763563.581501:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763563.581505:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763563.581509:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832637440, portal 10
00000100:00000001:1.0:1787763563.581511:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763563.581512:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
00000100:00000040:1.0:1787763563.581519:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832637440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763588 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.581525:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763563.581528:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763563.581530:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763563.581533:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832637440, offset 0
00000400:00000200:1.0:1787763563.581537:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763563.581547:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763563.581551:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763563.581555:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763563.581558:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763563.581561:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763563.581563:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763563.581567:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58215:58215:256:4294967295] 192.168.203.110@tcp LPNI seq info [58215:58215:8:4294967295]
00000400:00000200:1.0:1787763563.581571:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763563.581578:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763563.581583:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763563.581587:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763563.581592:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763563.581595:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787763563.581600:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763563.581603:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1787763563.581607:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763563.581609:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763563.581612:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763563.581615:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763563.581619:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763563.581621:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763563.581623:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763563.581627:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876ed00 x1874599832637440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763588 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.581635:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763563.581638:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832637440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763588 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.581644:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763563.581645:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763563.581649:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763563.581654:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763563.581657:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763563.581659:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315950624
00000100:00000001:1.0:1787763563.581662:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315951859
00000100:00000001:1.0:1787763563.581664:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315950624
00000100:00000001:1.0:1787763563.581666:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315951859
00000100:00000001:1.0:1787763563.581668:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763563.581670:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763563.581672:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763563.581675:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763563/real 1787763563]  req@ffff9751f876ed00 x1874599832637440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763563.581683:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763563.581684:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763563.581685:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763563.581688:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763563.581689:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763563.581691:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763563.581694:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832637440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763563.581702:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832637440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.581709:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763563.581710:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763563.581712:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763563.581713:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763563.581716:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832637440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.581723:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763563.581727:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832637440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.581733:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763563.581741:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763563.581743:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763563.581745:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763563.581746:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763563.581759:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763563.581762:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763563.581766:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763563.581768:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763563.581773:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832637440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763563.581782:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832637440:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763563.581786:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763563.581789:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832637568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.581796:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763563.581798:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763563.581800:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763563.581802:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763563.581804:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763563.581807:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832637568:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763563.581810:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763563.581812:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763563.581814:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763563.581816:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763563.581820:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763563.581822:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763563.581826:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763563.581828:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763563.581831:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763563.581833:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832637568, portal 10
00000100:00000001:1.0:1787763563.581835:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763563.581836:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000100:00000040:1.0:1787763563.581841:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832637568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763588 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.581846:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763563.581848:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763563.581849:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763563.581851:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832637568, offset 0
00000400:00000200:1.0:1787763563.581853:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763563.581856:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763563.581859:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763563.581861:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763563.581863:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763563.581864:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763563.581867:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763563.581870:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58216:58216:256:4294967295] 192.168.203.110@tcp LPNI seq info [58216:58216:8:4294967295]
00000400:00000200:1.0:1787763563.581875:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763563.581882:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763563.581887:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763563.581890:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763563.581894:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763563.581897:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787763563.581902:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763563.581905:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1787763563.581909:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763563.581911:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763563.581914:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763563.581918:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763563.581921:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763563.581923:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763563.581925:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763563.581929:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e1c0 x1874599832637568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763588 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.581939:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763563.581943:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832637568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763588 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.581951:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763563.581952:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763563.581956:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763563.581958:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763563.581960:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763563.581962:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315950612
00000100:00000001:1.0:1787763563.581964:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315951859
00000100:00000001:1.0:1787763563.581965:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315950612
00000100:00000001:1.0:1787763563.581967:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315951859
00000100:00000001:1.0:1787763563.581968:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763563.581969:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763563.581971:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763563.581973:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763563/real 1787763563]  req@ffff9751f876e1c0 x1874599832637568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763563.581979:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763563.581980:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763563.581981:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763563.581983:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763563.581984:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763563.581985:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763563.581987:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832637568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763563.581992:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832637568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.581996:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763563.581997:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763563.581998:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763563.581999:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763563.582002:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832637568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.582007:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763563.582010:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832637568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.582014:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763563.582016:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763563.582017:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763563.582018:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763563.582019:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763563.582020:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763563.582022:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763563.582024:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763563.582025:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763563.582027:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832637568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763563.582036:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832637568:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763563.582040:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763563.582043:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832637696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.582050:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763563.582051:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763563.582054:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763563.582055:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763563.582058:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763563.582061:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832637696:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763563.582064:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763563.582066:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763563.582068:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763563.582070:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763563.582074:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763563.582076:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763563.582080:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763563.582082:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763563.582085:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763563.582089:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832637696, portal 4
00000100:00000001:1.0:1787763563.582091:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763563.582092:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
00000100:00000040:1.0:1787763563.582099:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832637696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763588 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.582107:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763563.582109:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763563.582112:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763563.582115:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832637696, offset 0
00000400:00000200:1.0:1787763563.582118:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763563.582121:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763563.582124:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763563.582126:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763563.582128:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763563.582130:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763563.582132:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763563.582151:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58217:58217:256:4294967295] 192.168.203.110@tcp LPNI seq info [58217:58217:8:4294967295]
00000400:00000200:1.0:1787763563.582155:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763563.582159:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763563.582163:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763563.582165:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763563.582168:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763563.582170:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787763563.582172:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763563.582174:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1787763563.582177:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763563.582179:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763563.582181:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763563.582184:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763563.582186:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763563.582187:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763563.582189:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763563.582191:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876c000 x1874599832637696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763588 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.582197:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763563.582199:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832637696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763588 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.582203:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763563.582204:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763563.582207:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763563.582209:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763563.582211:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763563.582212:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315950630
00000100:00000001:1.0:1787763563.582214:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315951859
00000100:00000001:1.0:1787763563.582215:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315950630
00000100:00000001:1.0:1787763563.582217:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315951859
00000100:00000001:1.0:1787763563.582219:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763563.582220:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763563.582221:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763563.582224:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763563/real 1787763563]  req@ffff9751f876c000 x1874599832637696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763563.582228:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763563.582229:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763563.582230:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763563.582232:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763563.582233:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763563.582235:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763563.582252:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832637696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763563.582259:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832637696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.582263:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763563.582264:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763563.582265:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763563.582266:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763563.582268:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832637696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.582273:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763563.582276:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832637696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.582280:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763563.582282:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763563.582283:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763563.582283:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763563.582285:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763563.582286:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763563.582288:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763563.582291:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763563.582293:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763563.582297:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832637696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763563.582305:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832637696:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763563.582309:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763563.582311:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763563.582316:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832637440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.582323:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763563.582325:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763563.582327:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763563.582329:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763563.582331:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763563.582332:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763563.582335:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763563.582337:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:1.0:1787763563.582340:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763563.582341:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763563.582342:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:1.0:1787763563.582344:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763563.582346:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763563.582347:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763563.582350:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832637568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.582354:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763563.582355:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763563.582356:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763563.582358:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763563.582358:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763563.582359:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763563.582360:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763563.582361:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017800.
02000000:00000001:1.0:1787763563.582364:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763563.582364:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763563.582365:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:1.0:1787763563.582366:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763563.582367:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763563.582368:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763563.582370:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832637696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763588 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763563.582374:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763563.582376:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763563.582377:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763563.582378:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763563.582379:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763563.582380:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763563.582381:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763563.582382:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016c00.
02000000:00000001:1.0:1787763563.582384:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763563.582385:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763563.582385:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:1.0:1787763563.582387:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763563.582388:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763563.582389:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763563.582392:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763563.582393:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763563.582394:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763563.582396:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763564.220371:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763564.220382:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763564.220386:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763564.220393:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000020:00000001:0.0:1787763564.220396:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000100:00000001:1.0:1787763564.220401:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1787763564.220403:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763564.220410:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00010000:0.0:1787763564.220411:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763564.220421:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763564.220425:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763564.220430:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763565.244311:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763565.244332:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763565.244338:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763565.244346:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763565.244357:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763565.244365:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763565.244372:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763565.244375:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763565.244379:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763566.268322:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763566.268335:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763566.268341:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763566.268352:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763566.268369:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763566.268380:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763566.268391:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763566.268401:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763566.268413:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763566.268482:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763566.268489:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763566.268494:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763567.292207:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763567.292226:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763567.292231:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763567.292236:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763567.292243:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763567.292264:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763567.292268:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763567.292271:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763567.292273:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763567.292312:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763567.292320:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763567.292324:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763568.316322:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763568.316327:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763568.316331:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763568.700287:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763568.700292:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763568.700294:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763568.700298:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763568.700301:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763568.700304:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763568.700307:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763568.700310:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763568.700313:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763568.700430:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763568.700436:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763568.700440:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763568.700450:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832637824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.700468:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763568.700470:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763568.700475:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763568.700479:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763568.700486:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763568.700493:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832637824:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763568.700500:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763568.700503:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763568.700507:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763568.700511:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763568.700517:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763568.700522:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763568.700530:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763568.700536:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763568.700545:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763568.700552:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832637824, portal 25
00000100:00000001:1.0:1787763568.700555:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763568.700558:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:1.0:1787763568.700564:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832637824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763593 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.700573:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763568.700580:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763568.700584:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763568.700589:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832637824, offset 0
00000400:00000200:1.0:1787763568.700597:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763568.700609:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763568.700618:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763568.700627:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763568.700634:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763568.700639:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763568.700643:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763568.700649:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58218:58218:256:4294967295] 192.168.203.110@tcp LPNI seq info [58218:58218:8:4294967295]
00000400:00000200:1.0:1787763568.700657:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763568.700667:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763568.700674:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763568.700681:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763568.700694:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763568.700700:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787763568.700711:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763568.700718:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1787763568.700726:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763568.700731:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763568.700739:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763568.700749:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763568.700754:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763568.700760:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763568.700767:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763568.700773:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832637824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763593 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.700787:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763568.700792:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832637824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763593 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.700812:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763568.700818:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763568.700828:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763568.700836:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763568.700841:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315956979
00000100:00000010:1.0:1787763568.700852:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4ae0.
00000100:00000001:1.0:1787763568.700862:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787763568.700871:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315956979
00000100:00000010:1.0:1787763568.700876:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4840.
00000100:00000001:1.0:1787763568.700881:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763568.700885:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763568.700890:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763568.700895:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763568.700904:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763568/real 1787763568]  req@ffff9751f865ad00 x1874599832637824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763568.700922:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763568.700926:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763568.700929:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763568.700934:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763568.700938:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763568.700944:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763568.700951:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832637824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763568.700972:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832637824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.700990:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763568.700995:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763568.700999:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763568.701003:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763568.701009:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832637824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701022:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763568.701030:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832637824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701048:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763568.701051:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763568.701054:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763568.701055:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763568.701059:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763568.701061:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763568.701064:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763568.701072:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763568.701076:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763568.701084:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832637824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763568.701107:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832637824:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763568.701115:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763568.701119:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763568.701122:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832637824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701128:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763568.701130:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763568.701149:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763568.701152:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763568.701154:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763568.701155:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763568.701157:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763568.701159:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:1.0:1787763568.701162:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763568.701163:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763568.701164:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:1.0:1787763568.701167:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763568.701168:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763568.701169:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763568.701174:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763568.701175:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763568.701177:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763568.701178:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763568.701179:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763568.701182:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832637952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701187:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763568.701188:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763568.701189:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763568.701191:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763568.701192:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763568.701194:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832637952:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763568.701196:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763568.701198:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763568.701199:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763568.701216:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763568.701219:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763568.701221:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763568.701224:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763568.701226:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763568.701228:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763568.701231:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832637952, portal 10
00000100:00000001:1.0:1787763568.701232:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763568.701233:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787763568.701238:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832637952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763593 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701243:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763568.701244:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763568.701246:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763568.701247:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832637952, offset 0
00000400:00000200:1.0:1787763568.701250:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763568.701253:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763568.701255:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763568.701258:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763568.701259:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763568.701261:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763568.701263:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763568.701265:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58219:58219:256:4294967295] 192.168.203.110@tcp LPNI seq info [58219:58219:8:4294967295]
00000400:00000200:1.0:1787763568.701268:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763568.701272:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763568.701275:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763568.701278:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763568.701280:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763568.701282:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787763568.701285:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763568.701287:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1787763568.701289:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763568.701291:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763568.701293:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763568.701295:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763568.701297:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763568.701298:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763568.701300:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763568.701302:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832637952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763593 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701308:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763568.701310:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832637952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763593 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701319:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763568.701320:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763568.701322:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763568.701325:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763568.701327:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763568.701328:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315955744
00000100:00000001:1.0:1787763568.701330:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315956979
00000100:00000001:1.0:1787763568.701332:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315955744
00000100:00000001:1.0:1787763568.701333:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315956979
00000100:00000001:1.0:1787763568.701334:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763568.701335:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763568.701336:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763568.701339:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763568/real 1787763568]  req@ffff9751f8658000 x1874599832637952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763568.701344:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763568.701345:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763568.701346:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763568.701347:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763568.701348:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763568.701349:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763568.701351:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832637952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763568.701356:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832637952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701360:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763568.701361:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763568.701362:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763568.701363:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763568.701366:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832637952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701371:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763568.701373:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832637952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701378:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763568.701379:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763568.701380:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763568.701381:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763568.701382:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763568.701392:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763568.701393:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763568.701395:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763568.701396:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763568.701399:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832637952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763568.701403:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832637952:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763568.701406:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763568.701408:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832638080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701413:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763568.701414:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763568.701415:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763568.701416:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763568.701417:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763568.701419:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832638080:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763568.701421:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763568.701422:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763568.701424:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763568.701425:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763568.701427:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763568.701428:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763568.701430:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763568.701432:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763568.701434:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763568.701435:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832638080, portal 10
00000100:00000001:1.0:1787763568.701437:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763568.701438:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787763568.701442:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832638080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763593 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701446:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763568.701447:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763568.701449:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763568.701450:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832638080, offset 0
00000400:00000200:1.0:1787763568.701453:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763568.701455:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763568.701456:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763568.701458:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763568.701460:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763568.701461:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763568.701463:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763568.701465:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58220:58220:256:4294967295] 192.168.203.110@tcp LPNI seq info [58220:58220:8:4294967295]
00000400:00000200:1.0:1787763568.701467:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763568.701471:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763568.701474:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763568.701476:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763568.701478:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763568.701480:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787763568.701482:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763568.701484:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1787763568.701486:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763568.701488:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763568.701490:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763568.701492:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763568.701494:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763568.701495:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763568.701497:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763568.701499:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832638080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763593 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701504:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763568.701506:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832638080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763593 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701510:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763568.701511:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763568.701512:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763568.701514:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763568.701516:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763568.701517:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315955732
00000100:00000001:1.0:1787763568.701518:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315956979
00000100:00000001:1.0:1787763568.701520:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315955732
00000100:00000001:1.0:1787763568.701521:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315956979
00000100:00000001:1.0:1787763568.701522:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763568.701524:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763568.701525:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763568.701527:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763568/real 1787763568]  req@ffff9751f8659a40 x1874599832638080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763568.701531:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763568.701532:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763568.701533:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763568.701534:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763568.701535:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763568.701537:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763568.701539:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832638080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763568.701544:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832638080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701548:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763568.701549:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763568.701550:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763568.701550:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763568.701552:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832638080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701556:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763568.701559:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832638080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701563:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763568.701564:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763568.701565:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763568.701566:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763568.701567:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763568.701568:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763568.701569:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763568.701571:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763568.701573:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763568.701575:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832638080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763568.701580:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832638080:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763568.701582:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763568.701585:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832638208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701589:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763568.701590:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763568.701591:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763568.701592:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763568.701593:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763568.701595:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832638208:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763568.701597:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763568.701598:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763568.701599:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763568.701600:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763568.701602:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763568.701603:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763568.701605:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763568.701607:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763568.701608:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763568.701610:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832638208, portal 4
00000100:00000001:1.0:1787763568.701611:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763568.701612:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000040:1.0:1787763568.701616:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832638208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763593 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701620:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763568.701621:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763568.701623:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763568.701624:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832638208, offset 0
00000400:00000200:1.0:1787763568.701627:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763568.701628:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763568.701630:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763568.701632:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763568.701633:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763568.701635:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763568.701636:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763568.701639:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58221:58221:256:4294967295] 192.168.203.110@tcp LPNI seq info [58221:58221:8:4294967295]
00000400:00000200:1.0:1787763568.701641:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763568.701645:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763568.701648:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763568.701650:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763568.701652:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763568.701653:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787763568.701656:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763568.701657:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1787763568.701660:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763568.701661:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763568.701663:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763568.701665:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763568.701667:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763568.701668:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763568.701669:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763568.701671:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832638208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763593 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701676:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763568.701678:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832638208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763593 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701681:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763568.701682:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763568.701684:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763568.701686:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763568.701688:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763568.701689:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315955749
00000100:00000001:1.0:1787763568.701690:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315956979
00000100:00000001:1.0:1787763568.701692:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315955749
00000100:00000001:1.0:1787763568.701693:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315956979
00000100:00000001:1.0:1787763568.701694:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763568.701696:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763568.701697:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763568.701699:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763568/real 1787763568]  req@ffff9751f8658780 x1874599832638208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763568.701704:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763568.701705:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763568.701706:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763568.701707:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763568.701708:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763568.701709:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763568.701711:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832638208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763568.701716:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832638208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701720:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763568.701721:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763568.701722:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763568.701723:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763568.701725:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832638208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701729:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763568.701731:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832638208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701735:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763568.701736:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763568.701737:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763568.701738:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763568.701739:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763568.701740:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763568.701742:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763568.701744:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763568.701745:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763568.701748:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832638208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763568.701753:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832638208:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763568.701755:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763568.701756:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763568.701758:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832637952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701762:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763568.701763:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763568.701764:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763568.701765:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763568.701766:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763568.701767:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763568.701768:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763568.701770:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:1.0:1787763568.701771:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763568.701772:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763568.701773:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:1.0:1787763568.701774:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763568.701775:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763568.701776:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763568.701778:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832638080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701782:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763568.701783:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763568.701784:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763568.701785:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763568.701786:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763568.701787:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763568.701788:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763568.701789:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:1.0:1787763568.701791:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763568.701792:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763568.701793:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787763568.701796:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763568.701797:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763568.701799:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763568.701802:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832638208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763593 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763568.701808:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763568.701810:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763568.701811:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763568.701813:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763568.701814:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763568.701816:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763568.701818:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763568.701819:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:1.0:1787763568.701822:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763568.701823:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763568.701824:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:1.0:1787763568.701827:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763568.701828:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763568.701830:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763568.701833:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763568.701835:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763568.701837:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763568.701839:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763569.340311:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763569.340314:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763569.340316:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763569.724203:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763569.724212:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763569.724214:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763569.724219:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763569.724222:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763569.724225:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763569.724227:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763569.724229:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763569.724231:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763570.748287:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763570.748294:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763570.748297:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763570.748303:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763570.748306:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763570.748310:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763570.748313:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763570.748316:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763570.748318:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763571.388291:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763571.388298:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763571.388307:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763571.772212:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763571.772220:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763571.772222:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763571.772227:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763571.772231:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763571.772235:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763571.772237:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763571.772240:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763571.772242:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763572.412422:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763572.412426:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763572.412429:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763572.796262:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763572.796268:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763572.796270:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763572.796275:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763572.796277:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763572.796280:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763572.796282:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763572.796284:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763572.796286:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763573.820251:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763573.820256:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763573.820259:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763573.820264:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763573.820268:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763573.820271:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763573.820275:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763573.820279:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763573.820282:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763573.820432:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763573.820438:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763573.820440:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763573.820447:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832638336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.820456:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763573.820457:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763573.820461:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763573.820463:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763573.820467:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763573.820471:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832638336:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763573.820475:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763573.820479:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763573.820482:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763573.820484:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763573.820490:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763573.820492:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763573.820497:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763573.820501:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763573.820508:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763573.820515:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832638336, portal 25
00000100:00000001:1.0:1787763573.820517:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763573.820519:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:1.0:1787763573.820525:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832638336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763598 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.820534:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763573.820537:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763573.820540:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763573.820544:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832638336, offset 0
00000400:00000200:1.0:1787763573.820552:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763573.820560:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763573.820566:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763573.820572:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763573.820576:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763573.820580:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763573.820583:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763573.820587:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58222:58222:256:4294967295] 192.168.203.110@tcp LPNI seq info [58222:58222:8:4294967295]
00000400:00000200:1.0:1787763573.820593:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763573.820603:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763573.820609:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763573.820615:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763573.820622:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763573.820627:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787763573.820633:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763573.820637:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:1.0:1787763573.820642:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763573.820646:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763573.820651:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763573.820656:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763573.820661:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763573.820664:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763573.820667:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763573.820672:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832638336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763598 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.820682:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763573.820685:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832638336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763598 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.820694:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763573.820696:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763573.820702:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763573.820706:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763573.820709:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315962099
00000100:00000010:1.0:1787763573.820714:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4720.
00000100:00000001:1.0:1787763573.820721:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787763573.820727:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315962099
00000100:00000010:1.0:1787763573.820731:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c45a0.
00000100:00000001:1.0:1787763573.820734:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763573.820737:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763573.820740:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763573.820742:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763573.820756:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763573/real 1787763573]  req@ffff9751f86583c0 x1874599832638336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763573.820766:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763573.820769:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763573.820771:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763573.820774:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763573.820776:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763573.820779:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763573.820783:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832638336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763573.820793:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832638336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.820801:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763573.820802:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763573.820805:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763573.820807:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763573.820811:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832638336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.820819:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763573.820825:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832638336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.820833:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763573.820836:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763573.820838:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763573.820840:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763573.820843:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763573.820844:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763573.820847:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763573.820850:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763573.820852:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763573.820855:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832638336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763573.820861:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832638336:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763573.820865:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763573.820867:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763573.820869:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832638336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.820873:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763573.820875:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763573.820877:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763573.820880:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763573.820881:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763573.820882:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763573.820884:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763573.820886:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:1.0:1787763573.820889:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763573.820890:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763573.820892:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:1.0:1787763573.820894:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763573.820895:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763573.820896:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763573.820901:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763573.820902:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763573.820904:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763573.820905:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763573.820906:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763573.820908:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832638464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.820913:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763573.820914:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763573.820916:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763573.820917:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763573.820919:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763573.820920:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832638464:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763573.820922:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763573.820924:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763573.820925:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763573.820926:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763573.820929:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763573.820930:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763573.820933:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763573.820935:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763573.820938:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763573.820940:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832638464, portal 10
00000100:00000001:1.0:1787763573.820941:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763573.820942:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000040:1.0:1787763573.820947:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832638464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763598 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.820951:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763573.820953:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763573.820954:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763573.820956:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832638464, offset 0
00000400:00000200:1.0:1787763573.820959:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763573.820961:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763573.820964:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763573.820966:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763573.820968:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763573.820969:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763573.820971:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763573.820973:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58223:58223:256:4294967295] 192.168.203.110@tcp LPNI seq info [58223:58223:8:4294967295]
00000400:00000200:1.0:1787763573.820976:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763573.820981:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763573.820984:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763573.820987:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763573.820989:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763573.820991:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787763573.820995:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763573.820997:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:1.0:1787763573.820999:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763573.821001:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763573.821010:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763573.821013:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763573.821016:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763573.821017:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763573.821019:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763573.821021:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832638464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763598 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821027:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763573.821029:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832638464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763598 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821033:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763573.821034:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763573.821036:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763573.821039:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763573.821041:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763573.821042:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315960863
00000100:00000001:1.0:1787763573.821045:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315962099
00000100:00000001:1.0:1787763573.821046:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315960863
00000100:00000001:1.0:1787763573.821048:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315962099
00000100:00000001:1.0:1787763573.821049:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763573.821050:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763573.821052:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763573.821054:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763573/real 1787763573]  req@ffff9751f8658780 x1874599832638464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763573.821059:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763573.821060:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763573.821061:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763573.821062:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763573.821063:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763573.821065:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763573.821067:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832638464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763573.821073:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832638464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821077:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763573.821078:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763573.821079:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763573.821080:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763573.821082:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832638464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821086:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763573.821088:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832638464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821092:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763573.821094:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763573.821095:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763573.821095:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763573.821097:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763573.821107:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763573.821108:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763573.821110:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763573.821111:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763573.821114:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832638464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763573.821119:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832638464:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763573.821122:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763573.821124:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832638592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821148:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763573.821149:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763573.821151:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763573.821152:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763573.821154:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763573.821156:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832638592:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763573.821158:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763573.821160:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763573.821161:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763573.821162:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763573.821177:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763573.821178:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763573.821180:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763573.821182:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763573.821184:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763573.821186:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832638592, portal 10
00000100:00000001:1.0:1787763573.821188:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763573.821189:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787763573.821193:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832638592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763598 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821197:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763573.821199:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763573.821200:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763573.821202:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832638592, offset 0
00000400:00000200:1.0:1787763573.821204:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763573.821206:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763573.821208:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763573.821210:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763573.821212:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763573.821214:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763573.821216:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763573.821218:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58224:58224:256:4294967295] 192.168.203.110@tcp LPNI seq info [58224:58224:8:4294967295]
00000400:00000200:1.0:1787763573.821221:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763573.821225:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763573.821228:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763573.821230:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763573.821233:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763573.821235:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787763573.821237:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763573.821239:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:1.0:1787763573.821242:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763573.821243:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763573.821245:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763573.821247:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763573.821249:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763573.821250:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763573.821252:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763573.821254:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832638592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763598 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821260:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763573.821262:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832638592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763598 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821266:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763573.821267:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763573.821269:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763573.821271:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763573.821272:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763573.821274:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315960852
00000100:00000001:1.0:1787763573.821275:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315962099
00000100:00000001:1.0:1787763573.821277:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315960852
00000100:00000001:1.0:1787763573.821279:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315962099
00000100:00000001:1.0:1787763573.821280:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763573.821282:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763573.821283:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763573.821285:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763573/real 1787763573]  req@ffff9751f8659a40 x1874599832638592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763573.821291:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763573.821292:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763573.821293:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763573.821294:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763573.821295:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763573.821296:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763573.821298:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832638592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763573.821304:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832638592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821308:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763573.821309:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763573.821310:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763573.821310:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763573.821313:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832638592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821316:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763573.821319:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832638592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821323:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763573.821324:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763573.821325:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763573.821326:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763573.821327:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763573.821329:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763573.821330:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763573.821333:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763573.821334:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763573.821337:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832638592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763573.821342:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832638592:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763573.821344:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763573.821346:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832638720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821350:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763573.821351:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763573.821352:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763573.821353:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763573.821355:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763573.821356:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832638720:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763573.821358:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763573.821359:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763573.821361:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763573.821362:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763573.821364:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763573.821366:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763573.821368:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763573.821370:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763573.821372:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763573.821374:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832638720, portal 4
00000100:00000001:1.0:1787763573.821375:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763573.821376:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787763573.821380:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832638720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763598 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821384:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763573.821386:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763573.821388:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763573.821389:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832638720, offset 0
00000400:00000200:1.0:1787763573.821392:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763573.821394:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763573.821396:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763573.821397:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763573.821399:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763573.821400:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763573.821402:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763573.821404:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58225:58225:256:4294967295] 192.168.203.110@tcp LPNI seq info [58225:58225:8:4294967295]
00000400:00000200:1.0:1787763573.821407:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763573.821411:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763573.821413:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763573.821415:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763573.821417:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763573.821419:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:1.1:1787763573.821421:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763573.821423:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:1.0:1787763573.821425:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763573.821427:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763573.821429:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763573.821431:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763573.821433:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763573.821434:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763573.821435:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763573.821438:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832638720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763598 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821443:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763573.821445:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832638720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763598 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821449:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763573.821450:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763573.821452:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763573.821454:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763573.821456:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763573.821457:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315960869
00000100:00000001:1.0:1787763573.821459:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315962099
00000100:00000001:1.0:1787763573.821461:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315960869
00000100:00000001:1.0:1787763573.821462:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315962099
00000100:00000001:1.0:1787763573.821463:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763573.821464:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763573.821466:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763573.821468:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763573/real 1787763573]  req@ffff9751f8658000 x1874599832638720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763573.821472:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763573.821473:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763573.821474:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763573.821475:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763573.821476:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763573.821477:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763573.821480:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832638720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763573.821485:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832638720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821489:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763573.821490:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763573.821491:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763573.821491:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763573.821493:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832638720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821497:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763573.821500:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832638720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821505:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763573.821506:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763573.821507:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763573.821508:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763573.821509:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763573.821510:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763573.821512:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763573.821513:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763573.821515:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763573.821517:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832638720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763573.821521:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832638720:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763573.821524:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763573.821525:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763573.821527:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832638464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821530:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763573.821531:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763573.821533:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763573.821534:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763573.821535:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763573.821536:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763573.821537:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763573.821538:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:1.0:1787763573.821539:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763573.821540:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763573.821541:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:1.0:1787763573.821543:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763573.821544:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763573.821545:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763573.821547:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832638592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821552:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763573.821552:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763573.821553:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763573.821555:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763573.821555:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763573.821556:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763573.821557:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763573.821558:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:1.0:1787763573.821559:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763573.821560:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763573.821560:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787763573.821562:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763573.821563:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763573.821563:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763573.821565:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832638720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763598 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763573.821569:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763573.821569:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763573.821570:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763573.821571:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763573.821572:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763573.821573:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763573.821574:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763573.821575:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:1.0:1787763573.821576:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763573.821576:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763573.821577:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:1.0:1787763573.821578:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763573.821579:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763573.821580:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763573.821582:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763573.821583:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763573.821584:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763573.821586:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763574.460378:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763574.460383:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763574.460386:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763575.484317:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763575.484321:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763575.484323:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763577.532286:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763577.532290:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763577.532293:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763578.556357:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763578.556362:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763578.556366:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763578.940471:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763578.940483:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763578.940489:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763578.940509:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bcf00 x1874599832638848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.940551:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763578.940557:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763578.940568:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763578.940574:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763578.940584:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763578.940603:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832638848:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763578.940624:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763578.940636:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763578.940642:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763578.940656:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763578.940666:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763578.940670:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763578.940683:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763578.940692:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763578.940705:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763578.940714:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832638848, portal 25
00000100:00000001:1.0:1787763578.940724:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763578.940734:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049849600 : -115096659702016 : ffff9751f87bcf00)
00000100:00000040:1.0:1787763578.940746:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bcf00 x1874599832638848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763603 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.940781:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763578.940788:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763578.940793:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763578.940802:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832638848, offset 0
00000400:00000200:1.0:1787763578.940816:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763578.940828:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763578.940843:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763578.940850:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763578.940861:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763578.940872:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763578.940883:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763578.940896:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58226:58226:256:4294967295] 192.168.203.110@tcp LPNI seq info [58226:58226:8:4294967295]
00000400:00000200:1.0:1787763578.940910:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763578.940941:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763578.940963:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763578.940972:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763578.940991:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763578.941004:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787763578.941019:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763578.941029:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1787763578.941041:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763578.941049:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763578.941062:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763578.941078:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763578.941090:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763578.941100:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763578.941116:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763578.941127:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bcf00 x1874599832638848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763603 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.941208:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763578.941218:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bcf00 x1874599832638848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763603 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.941251:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763578.941260:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763578.941269:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763578.941275:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763578.941279:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315967220
00000100:00000010:1.0:1787763578.941285:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4e40.
00000100:00000001:1.0:1787763578.941292:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787763578.941299:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315967220
00000100:00000010:1.0:1787763578.941302:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4060.
00000100:00000001:1.0:1787763578.941304:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763578.941307:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763578.941309:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763578.941312:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763578.941316:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763578/real 1787763578]  req@ffff9751f87bcf00 x1874599832638848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763578.941323:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763578.941325:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763578.941326:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763578.941328:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763578.941330:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763578.941332:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763578.941335:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bcf00 x1874599832638848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763578.941342:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bcf00 x1874599832638848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.941346:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763578.941347:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763578.941349:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763578.941351:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763578.941353:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bcf00 x1874599832638848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.941358:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763578.941361:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bcf00 x1874599832638848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.941366:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763578.941368:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763578.941370:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763578.941371:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763578.941374:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763578.941376:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763578.941378:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763578.941382:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763578.941384:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763578.941386:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bcf00 x1874599832638848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763578.941392:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832638848:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763578.941395:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763578.941397:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763578.941399:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bcf00 x1874599832638848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.941403:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763578.941405:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763578.941408:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763578.941410:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763578.941412:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763578.941414:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763578.941415:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763578.941418:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613c00.
02000000:00000001:1.0:1787763578.941422:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763578.941424:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763578.941426:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bcf00.
00000100:00000001:1.0:1787763578.941428:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763578.941429:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763578.941431:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763578.941438:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763578.941440:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763578.941441:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763578.941443:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763578.941444:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763578.941447:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc780 x1874599832638976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.941457:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763578.941458:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763578.941459:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763578.941462:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763578.941464:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763578.941467:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832638976:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763578.941471:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763578.941474:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763578.941476:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763578.941477:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763578.941481:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763578.941482:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763578.941486:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763578.941488:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763578.941491:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763578.941493:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832638976, portal 10
00000100:00000001:1.0:1787763578.941494:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763578.941496:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049847680 : -115096659703936 : ffff9751f87bc780)
00000100:00000040:1.0:1787763578.941504:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc780 x1874599832638976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763603 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.941512:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763578.941515:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763578.941517:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763578.941521:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832638976, offset 0
00000400:00000200:1.0:1787763578.941526:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763578.941530:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763578.941535:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763578.941538:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763578.941541:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763578.941544:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763578.941547:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763578.941551:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58227:58227:256:4294967295] 192.168.203.110@tcp LPNI seq info [58227:58227:8:4294967295]
00000400:00000200:1.0:1787763578.941567:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763578.941575:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763578.941580:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763578.941584:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763578.941589:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763578.941592:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787763578.941604:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763578.941607:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1787763578.941612:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763578.941614:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763578.941618:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763578.941622:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763578.941625:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763578.941627:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763578.941630:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763578.941634:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc780 x1874599832638976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763603 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.941643:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763578.941647:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc780 x1874599832638976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763603 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.941655:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763578.941656:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763578.941661:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763578.941665:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763578.941669:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763578.941671:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315965984
00000100:00000001:1.0:1787763578.941675:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315967220
00000100:00000001:1.0:1787763578.941677:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315965984
00000100:00000001:1.0:1787763578.941680:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315967220
00000100:00000001:1.0:1787763578.941682:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763578.941686:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763578.941687:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763578.941692:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763578/real 1787763578]  req@ffff9751f87bc780 x1874599832638976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763578.941702:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763578.941703:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763578.941706:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763578.941708:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763578.941710:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763578.941713:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763578.941717:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc780 x1874599832638976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763578.941727:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc780 x1874599832638976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.941735:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763578.941737:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763578.941739:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763578.941741:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763578.941745:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc780 x1874599832638976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.941753:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763578.941758:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc780 x1874599832638976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.941766:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763578.941769:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763578.941771:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763578.941773:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763578.941774:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763578.941788:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763578.941792:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763578.941797:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763578.941800:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763578.941804:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc780 x1874599832638976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763578.941813:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832638976:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763578.941819:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763578.941822:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc3c0 x1874599832639104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.941829:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763578.941830:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763578.941833:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763578.941835:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763578.941837:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763578.941840:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832639104:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763578.941844:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763578.941846:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763578.941848:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763578.941850:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763578.941854:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763578.941856:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763578.941860:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763578.941863:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763578.941866:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763578.941869:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832639104, portal 10
00000100:00000001:1.0:1787763578.941871:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763578.941873:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049846720 : -115096659704896 : ffff9751f87bc3c0)
00000100:00000040:1.0:1787763578.941879:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc3c0 x1874599832639104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763603 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.941887:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763578.941891:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763578.941893:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763578.941896:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832639104, offset 0
00000400:00000200:1.0:1787763578.941901:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763578.941906:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763578.941911:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763578.941914:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763578.941917:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763578.941920:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763578.941922:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763578.941926:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58228:58228:256:4294967295] 192.168.203.110@tcp LPNI seq info [58228:58228:8:4294967295]
00000400:00000200:1.0:1787763578.941931:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763578.941938:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763578.941943:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763578.941947:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763578.941951:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763578.941954:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787763578.941959:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763578.941962:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1787763578.941967:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763578.941969:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763578.941972:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763578.941976:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763578.941979:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763578.941981:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763578.941984:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763578.941988:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc3c0 x1874599832639104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763603 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.941997:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763578.942001:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc3c0 x1874599832639104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763603 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.942008:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763578.942010:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763578.942015:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763578.942018:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763578.942022:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763578.942024:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315965972
00000100:00000001:1.0:1787763578.942027:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315967220
00000100:00000001:1.0:1787763578.942030:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315965972
00000100:00000001:1.0:1787763578.942032:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315967220
00000100:00000001:1.0:1787763578.942034:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763578.942036:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763578.942038:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763578.942042:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763578/real 1787763578]  req@ffff9751f87bc3c0 x1874599832639104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763578.942051:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763578.942053:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763578.942055:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763578.942058:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763578.942060:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763578.942062:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763578.942066:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc3c0 x1874599832639104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763578.942076:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc3c0 x1874599832639104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.942084:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763578.942085:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763578.942087:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763578.942089:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763578.942093:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc3c0 x1874599832639104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.942101:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763578.942104:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc3c0 x1874599832639104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.942113:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763578.942116:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763578.942117:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763578.942119:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763578.942121:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763578.942123:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763578.942126:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763578.942169:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763578.942172:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763578.942177:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc3c0 x1874599832639104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763578.942186:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832639104:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763578.942191:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763578.942195:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bfc00 x1874599832639232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.942202:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763578.942203:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763578.942205:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763578.942207:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763578.942208:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763578.942210:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832639232:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763578.942213:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763578.942215:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763578.942216:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763578.942217:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763578.942221:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763578.942223:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763578.942226:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763578.942228:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763578.942231:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763578.942233:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832639232, portal 4
00000100:00000001:1.0:1787763578.942235:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763578.942236:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049861120 : -115096659690496 : ffff9751f87bfc00)
00000100:00000040:1.0:1787763578.942240:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bfc00 x1874599832639232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763603 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.942245:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763578.942247:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763578.942248:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763578.942250:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832639232, offset 0
00000400:00000200:1.0:1787763578.942253:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763578.942256:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763578.942258:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763578.942260:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763578.942262:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763578.942264:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763578.942266:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763578.942268:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58229:58229:256:4294967295] 192.168.203.110@tcp LPNI seq info [58229:58229:8:4294967295]
00000400:00000200:1.0:1787763578.942271:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763578.942275:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763578.942278:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763578.942280:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763578.942283:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763578.942285:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:1.1:1787763578.942288:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763578.942290:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:1.0:1787763578.942293:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763578.942294:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763578.942296:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763578.942299:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763578.942301:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763578.942303:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763578.942304:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763578.942307:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bfc00 x1874599832639232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763603 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.942312:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763578.942314:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bfc00 x1874599832639232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763603 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.942319:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763578.942319:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763578.942321:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763578.942323:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763578.942325:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763578.942326:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315965990
00000100:00000001:1.0:1787763578.942328:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315967220
00000100:00000001:1.0:1787763578.942329:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315965990
00000100:00000001:1.0:1787763578.942331:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315967220
00000100:00000001:1.0:1787763578.942332:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763578.942334:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763578.942335:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763578.942337:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763578/real 1787763578]  req@ffff9751f87bfc00 x1874599832639232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763578.942342:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763578.942343:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763578.942344:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763578.942345:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763578.942346:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763578.942347:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763578.942349:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bfc00 x1874599832639232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763578.942354:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bfc00 x1874599832639232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.942359:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763578.942361:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763578.942362:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763578.942363:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763578.942366:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bfc00 x1874599832639232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.942369:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763578.942372:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bfc00 x1874599832639232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.942376:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763578.942377:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763578.942378:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763578.942379:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763578.942380:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763578.942381:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763578.942382:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763578.942384:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763578.942386:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763578.942388:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bfc00 x1874599832639232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763578.942393:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832639232:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763578.942396:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763578.942397:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763578.942399:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc780 x1874599832638976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.942403:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763578.942404:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763578.942406:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763578.942408:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763578.942409:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763578.942410:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763578.942411:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763578.942413:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610400.
02000000:00000001:1.0:1787763578.942415:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763578.942415:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763578.942416:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc780.
00000100:00000001:1.0:1787763578.942418:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763578.942419:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763578.942420:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763578.942422:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc3c0 x1874599832639104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.942426:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763578.942427:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763578.942428:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763578.942429:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763578.942430:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763578.942431:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763578.942432:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763578.942433:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611800.
02000000:00000001:1.0:1787763578.942434:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763578.942435:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763578.942435:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc3c0.
00000100:00000001:1.0:1787763578.942437:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763578.942437:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763578.942438:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763578.942440:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bfc00 x1874599832639232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763603 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763578.942444:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763578.942444:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763578.942445:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763578.942446:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763578.942447:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763578.942448:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763578.942449:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763578.942450:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613800.
02000000:00000001:1.0:1787763578.942452:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763578.942453:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763578.942454:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bfc00.
00000100:00000001:1.0:1787763578.942455:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763578.942456:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763578.942457:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763578.942459:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763578.942460:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763578.942461:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763578.942463:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763580.604343:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763580.604347:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763580.604349:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763581.628297:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763581.628300:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763581.628303:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763582.012265:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763582.012270:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763582.012272:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763582.012276:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763582.012279:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763582.012282:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763582.012285:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763582.012287:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763582.012290:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763583.036218:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763583.036224:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763583.036226:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763583.036230:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763583.036234:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763583.036237:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763583.036239:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763583.036241:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763583.036243:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763583.676265:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763583.676268:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763583.676271:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763584.060249:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763584.060256:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763584.060259:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763584.060265:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763584.060271:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763584.060276:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763584.060281:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763584.060286:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763584.060293:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763584.060526:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763584.060532:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763584.060536:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763584.060545:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832639360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.060562:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763584.060564:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763584.060569:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763584.060572:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763584.060577:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763584.060583:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832639360:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763584.060597:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763584.060603:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763584.060606:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763584.060608:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763584.060613:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763584.060618:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763584.060625:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763584.060629:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763584.060639:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763584.060648:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832639360, portal 25
00000100:00000001:1.0:1787763584.060654:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763584.060658:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:1.0:1787763584.060671:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832639360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763609 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.060694:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763584.060701:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763584.060707:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763584.060715:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832639360, offset 0
00000400:00000200:1.0:1787763584.060728:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763584.060737:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763584.060747:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763584.060753:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763584.060758:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763584.060764:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763584.060769:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763584.060775:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58230:58230:256:4294967295] 192.168.203.110@tcp LPNI seq info [58230:58230:8:4294967295]
00000400:00000200:1.0:1787763584.060782:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763584.060795:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763584.060804:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763584.060812:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763584.060821:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763584.060826:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787763584.060835:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763584.060840:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:1.0:1787763584.060847:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763584.060850:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763584.060855:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763584.060864:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763584.060870:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763584.060874:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763584.060879:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763584.060885:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832639360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763609 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.060900:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763584.060905:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832639360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763609 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.060916:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763584.060922:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763584.060929:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763584.060934:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763584.060939:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315972339
00000100:00000010:1.0:1787763584.060944:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4b40.
00000100:00000001:1.0:1787763584.060952:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787763584.060962:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315972339
00000100:00000010:1.0:1787763584.060967:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4f60.
00000100:00000001:1.0:1787763584.060971:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763584.060976:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763584.060979:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763584.060982:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763584.060989:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763584/real 1787763584]  req@ffff9751f865ad00 x1874599832639360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763584.061004:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763584.061007:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763584.061010:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763584.061012:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763584.061015:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763584.061018:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763584.061024:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832639360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763584.061044:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832639360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061056:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763584.061058:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.061063:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763584.061065:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763584.061070:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832639360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061082:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763584.061088:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832639360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061113:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763584.061119:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.061123:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763584.061125:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.061128:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763584.061131:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763584.061152:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763584.061156:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763584.061158:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763584.061162:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832639360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763584.061174:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832639360:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763584.061182:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.061185:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763584.061196:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832639360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061208:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763584.061212:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763584.061215:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763584.061219:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763584.061221:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763584.061224:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763584.061227:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763584.061231:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:1.0:1787763584.061235:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763584.061238:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763584.061240:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:1.0:1787763584.061245:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763584.061248:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.061250:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.061257:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763584.061260:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763584.061263:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763584.061266:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763584.061267:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763584.061272:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832639488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061284:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763584.061286:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763584.061290:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763584.061293:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763584.061297:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763584.061301:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832639488:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763584.061304:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763584.061307:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763584.061310:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763584.061313:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763584.061317:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763584.061320:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763584.061326:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763584.061330:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763584.061336:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763584.061339:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832639488, portal 10
00000100:00000001:1.0:1787763584.061344:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763584.061347:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787763584.061361:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832639488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763609 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061384:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763584.061389:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763584.061392:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763584.061396:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832639488, offset 0
00000400:00000200:1.0:1787763584.061404:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763584.061409:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763584.061414:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763584.061421:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763584.061425:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763584.061429:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763584.061432:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763584.061437:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58231:58231:256:4294967295] 192.168.203.110@tcp LPNI seq info [58231:58231:8:4294967295]
00000400:00000200:1.0:1787763584.061440:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763584.061449:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763584.061453:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763584.061456:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763584.061459:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763584.061461:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787763584.061464:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763584.061466:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:1.0:1787763584.061469:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763584.061470:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763584.061472:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763584.061474:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763584.061476:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763584.061478:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763584.061479:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763584.061482:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832639488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763609 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061487:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763584.061489:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832639488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763609 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061493:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763584.061494:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763584.061497:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763584.061500:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763584.061502:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763584.061503:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315971104
00000100:00000001:1.0:1787763584.061505:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315972339
00000100:00000001:1.0:1787763584.061507:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315971104
00000100:00000001:1.0:1787763584.061508:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315972339
00000100:00000001:1.0:1787763584.061510:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763584.061511:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763584.061512:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763584.061515:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763584/real 1787763584]  req@ffff9751f8658000 x1874599832639488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763584.061519:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763584.061520:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763584.061521:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763584.061522:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763584.061523:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763584.061525:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763584.061526:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832639488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763584.061532:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832639488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061536:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763584.061537:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.061538:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763584.061539:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763584.061541:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832639488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061545:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763584.061547:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832639488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061553:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763584.061554:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.061555:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763584.061556:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.061557:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763584.061568:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763584.061569:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763584.061571:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763584.061572:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763584.061575:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832639488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763584.061580:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832639488:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763584.061582:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763584.061584:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832639616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061588:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763584.061589:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763584.061590:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763584.061592:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763584.061593:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763584.061594:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832639616:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763584.061596:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763584.061597:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763584.061598:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763584.061599:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763584.061602:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763584.061603:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763584.061605:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763584.061607:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763584.061609:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763584.061610:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832639616, portal 10
00000100:00000001:1.0:1787763584.061612:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763584.061613:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787763584.061617:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832639616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763609 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061621:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763584.061623:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763584.061624:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763584.061626:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832639616, offset 0
00000400:00000200:1.0:1787763584.061628:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763584.061630:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763584.061632:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763584.061634:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763584.061636:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763584.061638:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763584.061641:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763584.061644:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58232:58232:256:4294967295] 192.168.203.110@tcp LPNI seq info [58232:58232:8:4294967295]
00000400:00000200:1.0:1787763584.061648:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763584.061655:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763584.061660:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763584.061663:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763584.061667:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763584.061670:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787763584.061675:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763584.061678:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:1.0:1787763584.061682:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763584.061684:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763584.061687:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763584.061690:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763584.061694:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763584.061696:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763584.061698:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763584.061701:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832639616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763609 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061707:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763584.061709:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832639616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763609 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061713:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763584.061715:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763584.061717:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763584.061719:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763584.061722:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763584.061723:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315971092
00000100:00000001:1.0:1787763584.061725:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315972339
00000100:00000001:1.0:1787763584.061727:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315971092
00000100:00000001:1.0:1787763584.061729:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315972339
00000100:00000001:1.0:1787763584.061730:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763584.061731:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763584.061732:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763584.061735:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763584/real 1787763584]  req@ffff9751f8659a40 x1874599832639616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763584.061740:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763584.061741:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763584.061743:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763584.061744:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763584.061745:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763584.061747:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763584.061749:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832639616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763584.061754:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832639616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061759:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763584.061760:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.061761:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763584.061762:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763584.061764:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832639616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061768:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763584.061771:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832639616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061775:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763584.061776:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.061777:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763584.061778:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.061779:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763584.061780:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763584.061781:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763584.061783:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763584.061784:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763584.061787:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832639616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763584.061791:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832639616:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763584.061794:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.061795:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763584.061797:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832639488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061801:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763584.061802:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763584.061804:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763584.061805:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763584.061806:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763584.061807:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763584.061809:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763584.061810:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:1.0:1787763584.061812:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763584.061813:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763584.061814:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:1.0:1787763584.061815:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763584.061816:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.061817:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763584.061819:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832639616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061823:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763584.061824:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763584.061825:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763584.061826:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763584.061826:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763584.061827:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763584.061828:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763584.061829:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:1.0:1787763584.061830:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763584.061831:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763584.061832:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787763584.061833:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763584.061834:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.061835:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.061837:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763584.061838:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763584.061839:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763584.061840:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763584.061841:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763584.061843:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832639744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061847:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763584.061848:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763584.061850:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763584.061851:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763584.061852:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763584.061854:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832639744:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763584.061856:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763584.061857:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763584.061858:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763584.061859:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763584.061862:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763584.061863:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763584.061865:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763584.061867:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763584.061869:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763584.061871:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832639744, portal 4
00000100:00000001:1.0:1787763584.061872:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763584.061873:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000040:1.0:1787763584.061877:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832639744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763609 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061881:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763584.061883:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763584.061884:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763584.061885:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832639744, offset 0
00000400:00000200:1.0:1787763584.061887:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763584.061890:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763584.061892:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763584.061894:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763584.061895:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763584.061897:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763584.061899:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763584.061901:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58233:58233:256:4294967295] 192.168.203.110@tcp LPNI seq info [58233:58233:8:4294967295]
00000400:00000200:1.0:1787763584.061903:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763584.061908:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763584.061911:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763584.061914:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763584.061916:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763584.061918:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:1.1:1787763584.061921:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763584.061923:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:1.0:1787763584.061925:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763584.061927:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763584.061929:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763584.061931:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763584.061933:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763584.061934:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763584.061936:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763584.061938:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832639744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763609 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061943:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763584.061945:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832639744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763609 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061949:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763584.061950:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763584.061952:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763584.061953:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763584.061956:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763584.061957:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315971109
00000100:00000001:1.0:1787763584.061959:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315972339
00000100:00000001:1.0:1787763584.061961:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315971109
00000100:00000001:1.0:1787763584.061962:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315972339
00000100:00000001:1.0:1787763584.061963:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763584.061964:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763584.061965:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763584.061968:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763584/real 1787763584]  req@ffff9751f8658780 x1874599832639744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763584.061973:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763584.061974:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763584.061975:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763584.061976:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763584.061977:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763584.061978:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763584.061980:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832639744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763584.061985:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832639744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061989:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763584.061990:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.061991:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763584.061991:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763584.061993:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832639744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.061998:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763584.062000:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832639744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.062004:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763584.062006:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.062007:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763584.062008:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.062009:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763584.062010:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763584.062011:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763584.062013:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763584.062014:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763584.062017:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832639744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763584.062022:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832639744:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763584.062024:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.062025:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763584.062027:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832639744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763609 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763584.062031:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763584.062032:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763584.062033:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763584.062034:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763584.062035:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763584.062035:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763584.062036:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763584.062037:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:1.0:1787763584.062039:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763584.062039:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763584.062040:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:1.0:1787763584.062042:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763584.062042:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.062043:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763584.062045:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763584.062046:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763584.062047:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763584.062048:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763584.700296:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763584.700300:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763584.700303:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763585.084217:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763585.084223:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763585.084224:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763585.084228:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763585.084232:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763585.084235:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763585.084236:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763585.084238:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763585.084240:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763586.108224:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763586.108235:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763586.108238:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763586.108245:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763586.108251:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763586.108255:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763586.108258:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763586.108262:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763586.108266:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763586.748318:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763586.748322:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763586.748325:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763587.132243:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763587.132253:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763587.132256:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763587.132264:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763587.132269:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763587.132272:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763587.132275:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763587.132277:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763587.132280:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763587.772344:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763587.772351:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763587.772357:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763588.156248:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763588.156256:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763588.156258:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763588.156264:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763588.156268:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763588.156272:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763588.156275:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763588.156278:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763588.156280:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763589.180506:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763589.180518:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763589.180526:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763589.180546:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832639872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.180577:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763589.180582:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763589.180595:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763589.180603:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763589.180614:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763589.180630:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832639872:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763589.180636:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763589.180643:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763589.180646:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763589.180664:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763589.180674:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763589.180680:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763589.180690:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763589.180697:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763589.180716:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763589.180728:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832639872, portal 25
00000100:00000001:1.0:1787763589.180738:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763589.180744:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:1.0:1787763589.180758:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832639872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763614 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.180780:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763589.180787:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763589.180791:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763589.180800:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832639872, offset 0
00000400:00000200:1.0:1787763589.180815:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763589.180830:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763589.180846:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763589.180858:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763589.180872:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763589.180884:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763589.180887:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763589.180892:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58234:58234:256:4294967295] 192.168.203.110@tcp LPNI seq info [58234:58234:8:4294967295]
00000400:00000200:1.0:1787763589.180898:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763589.180927:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763589.180956:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763589.180970:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763589.180993:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763589.181005:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787763589.181029:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763589.181038:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:1.0:1787763589.181057:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763589.181075:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763589.181084:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763589.181099:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763589.181107:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763589.181114:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763589.181123:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763589.181131:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832639872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763614 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.181171:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763589.181179:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832639872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763614 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.181194:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763589.181198:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763589.181207:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763589.181214:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763589.181219:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315977460
00000100:00000010:1.0:1787763589.181231:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c44e0.
00000100:00000001:1.0:1787763589.181244:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787763589.181254:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315977460
00000100:00000010:1.0:1787763589.181263:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c49c0.
00000100:00000001:1.0:1787763589.181280:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763589.181285:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763589.181288:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763589.181295:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763589.181304:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763589/real 1787763589]  req@ffff9751f86583c0 x1874599832639872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763589.181352:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763589.181360:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763589.181371:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763589.181386:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763589.181391:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763589.181399:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763589.181406:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832639872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763589.181474:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832639872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.181495:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763589.181500:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763589.181509:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763589.181515:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763589.181528:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832639872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.181541:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763589.181545:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832639872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.181550:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763589.181553:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763589.181554:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763589.181555:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763589.181557:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763589.181559:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763589.181562:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763589.181565:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763589.181567:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763589.181570:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832639872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763589.181576:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832639872:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763589.181580:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763589.181584:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763589.181586:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832639872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.181591:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763589.181594:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763589.181597:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763589.181599:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763589.181602:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763589.181604:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763589.181607:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763589.181609:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:1.0:1787763589.181614:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763589.181615:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763589.181617:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:1.0:1787763589.181620:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763589.181622:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763589.181624:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763589.181632:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763589.181633:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763589.181635:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763589.181636:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763589.181638:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763589.181641:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832640000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.181647:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763589.181648:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763589.181650:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763589.181652:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763589.181653:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763589.181655:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832640000:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763589.181659:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763589.181661:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763589.181662:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763589.181664:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763589.181666:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763589.181668:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763589.181670:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763589.181673:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763589.181676:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763589.181678:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832640000, portal 10
00000100:00000001:1.0:1787763589.181680:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763589.181681:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000040:1.0:1787763589.181686:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832640000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763614 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.181692:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763589.181695:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763589.181699:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763589.181703:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832640000, offset 0
00000400:00000200:1.0:1787763589.181711:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763589.181719:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763589.181726:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763589.181731:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763589.181736:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763589.181739:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763589.181743:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763589.181750:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58235:58235:256:4294967295] 192.168.203.110@tcp LPNI seq info [58235:58235:8:4294967295]
00000400:00000200:1.0:1787763589.181757:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763589.181768:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763589.181777:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763589.181781:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763589.181786:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763589.181790:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787763589.181799:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763589.181803:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:1.0:1787763589.181808:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763589.181810:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763589.181814:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763589.181822:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763589.181826:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763589.181828:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763589.181831:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763589.181837:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832640000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763614 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.181849:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763589.181855:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832640000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763614 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.181864:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763589.181866:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763589.181871:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763589.181875:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763589.181878:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763589.181880:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315976224
00000100:00000001:1.0:1787763589.181882:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315977460
00000100:00000001:1.0:1787763589.181884:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315976224
00000100:00000001:1.0:1787763589.181885:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315977460
00000100:00000001:1.0:1787763589.181887:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763589.181889:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763589.181890:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763589.181893:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763589/real 1787763589]  req@ffff9751f8658780 x1874599832640000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763589.181898:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763589.181899:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763589.181901:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763589.181902:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763589.181903:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763589.181905:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763589.181907:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832640000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763589.181913:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832640000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.181918:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763589.181919:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763589.181920:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763589.181921:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763589.181923:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832640000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.181927:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763589.181929:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832640000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.181933:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763589.181935:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763589.181936:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763589.181936:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763589.181938:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763589.181948:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763589.181949:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763589.181951:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763589.181953:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763589.181955:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832640000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763589.181960:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832640000:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763589.181963:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763589.181966:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832640128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.181971:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763589.181972:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763589.181973:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763589.181975:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763589.181976:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763589.181978:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832640128:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763589.181980:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763589.181982:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763589.181983:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763589.181984:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763589.181987:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763589.181989:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763589.181991:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763589.181993:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763589.181995:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763589.181997:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832640128, portal 10
00000100:00000001:1.0:1787763589.181999:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763589.182000:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787763589.182004:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832640128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763614 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.182009:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763589.182011:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763589.182012:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763589.182014:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832640128, offset 0
00000400:00000200:1.0:1787763589.182016:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763589.182019:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763589.182021:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763589.182023:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763589.182025:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763589.182027:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763589.182028:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763589.182031:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58236:58236:256:4294967295] 192.168.203.110@tcp LPNI seq info [58236:58236:8:4294967295]
00000400:00000200:1.0:1787763589.182034:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763589.182038:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763589.182041:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763589.182043:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763589.182046:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763589.182048:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787763589.182051:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763589.182053:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:1.0:1787763589.182056:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763589.182057:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763589.182071:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763589.182075:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763589.182077:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763589.182078:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763589.182080:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763589.182082:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832640128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763614 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.182087:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763589.182089:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832640128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763614 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.182094:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763589.182095:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763589.182097:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763589.182099:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763589.182101:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763589.182103:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315976212
00000100:00000001:1.0:1787763589.182105:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315977460
00000100:00000001:1.0:1787763589.182106:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315976212
00000100:00000001:1.0:1787763589.182108:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315977460
00000100:00000001:1.0:1787763589.182110:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763589.182111:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763589.182113:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763589.182115:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763589/real 1787763589]  req@ffff9751f8659a40 x1874599832640128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763589.182120:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763589.182120:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763589.182121:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763589.182123:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763589.182124:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763589.182125:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763589.182127:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832640128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763589.182149:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832640128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.182155:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763589.182156:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763589.182158:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763589.182161:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763589.182165:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832640128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.182173:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763589.182180:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832640128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.182185:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763589.182186:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763589.182187:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763589.182188:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763589.182190:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763589.182191:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763589.182192:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763589.182194:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763589.182196:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763589.182198:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832640128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763589.182203:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832640128:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763589.182206:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763589.182208:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832640256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.182212:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763589.182213:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763589.182214:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763589.182216:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763589.182217:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763589.182219:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832640256:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763589.182221:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763589.182223:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763589.182224:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763589.182226:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763589.182228:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763589.182230:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763589.182232:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763589.182234:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763589.182236:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763589.182238:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832640256, portal 4
00000100:00000001:1.0:1787763589.182240:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763589.182241:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787763589.182245:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832640256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763614 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.182250:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763589.182252:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763589.182253:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763589.182255:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832640256, offset 0
00000400:00000200:1.0:1787763589.182257:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763589.182259:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763589.182262:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763589.182264:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763589.182265:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763589.182267:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763589.182268:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763589.182271:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58237:58237:256:4294967295] 192.168.203.110@tcp LPNI seq info [58237:58237:8:4294967295]
00000400:00000200:1.0:1787763589.182273:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763589.182278:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763589.182280:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763589.182282:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763589.182284:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763589.182286:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:1.1:1787763589.182289:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763589.182291:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:1.0:1787763589.182294:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763589.182296:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763589.182298:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763589.182300:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763589.182302:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763589.182303:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763589.182304:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763589.182306:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832640256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763614 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.182311:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763589.182313:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832640256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763614 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.182323:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763589.182324:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763589.182326:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763589.182328:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763589.182330:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763589.182331:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315976230
00000100:00000001:1.0:1787763589.182333:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315977460
00000100:00000001:1.0:1787763589.182335:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315976230
00000100:00000001:1.0:1787763589.182336:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315977460
00000100:00000001:1.0:1787763589.182338:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763589.182339:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763589.182340:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763589.182342:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763589/real 1787763589]  req@ffff9751f8658000 x1874599832640256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763589.182346:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763589.182347:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763589.182348:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763589.182349:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763589.182350:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763589.182352:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763589.182354:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832640256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763589.182360:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832640256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.182363:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763589.182364:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763589.182365:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763589.182366:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763589.182368:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832640256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.182374:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763589.182377:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832640256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.182381:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763589.182382:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763589.182383:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763589.182384:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763589.182385:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763589.182386:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763589.182387:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763589.182389:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763589.182390:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763589.182393:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832640256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763589.182397:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832640256:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763589.182399:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763589.182401:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763589.182403:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832640000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.182406:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763589.182407:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763589.182409:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763589.182410:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763589.182411:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763589.182412:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763589.182414:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763589.182415:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:1.0:1787763589.182417:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763589.182419:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763589.182420:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:1.0:1787763589.182421:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763589.182422:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763589.182423:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763589.182426:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832640128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.182430:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763589.182431:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763589.182432:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763589.182433:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763589.182434:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763589.182434:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763589.182435:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763589.182436:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:1.0:1787763589.182438:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763589.182438:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763589.182439:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787763589.182441:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763589.182441:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763589.182442:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763589.182444:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832640256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763614 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763589.182448:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763589.182448:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763589.182449:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763589.182450:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763589.182451:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763589.182452:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763589.182453:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763589.182453:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:1.0:1787763589.182455:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763589.182455:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763589.182456:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:1.0:1787763589.182457:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763589.182459:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763589.182460:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763589.182463:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763589.182464:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763589.182466:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763589.182467:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763589.820395:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763589.820399:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763589.820401:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763590.844438:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763590.844442:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763590.844445:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763592.892301:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763592.892305:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763592.892307:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763593.916328:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763593.916332:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763593.916335:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763594.300558:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763594.300563:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763594.300566:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763594.300575:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832640384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.300591:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763594.300592:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763594.300596:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763594.300600:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763594.300604:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763594.300609:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832640384:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763594.300616:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763594.300619:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763594.300620:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763594.300623:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763594.300629:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763594.300633:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763594.300641:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763594.300648:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763594.300655:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763594.300663:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832640384, portal 25
00000100:00000001:1.0:1787763594.300668:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763594.300670:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:1.0:1787763594.300680:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832640384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763619 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.300692:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763594.300698:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763594.300714:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763594.300721:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832640384, offset 0
00000400:00000200:1.0:1787763594.300731:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763594.300741:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763594.300752:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763594.300760:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763594.300767:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763594.300783:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763594.300787:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763594.300792:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58238:58238:256:4294967295] 192.168.203.110@tcp LPNI seq info [58238:58238:8:4294967295]
00000400:00000200:1.0:1787763594.300795:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763594.300808:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763594.300818:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763594.300824:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763594.300835:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763594.300840:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787763594.300849:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763594.300855:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1787763594.300861:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763594.300863:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763594.300868:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763594.300874:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763594.300879:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763594.300882:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763594.300888:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763594.300893:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832640384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763619 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.300905:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763594.300910:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832640384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763619 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.300919:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763594.300921:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763594.300926:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763594.300933:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763594.300937:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315982579
00000100:00000010:1.0:1787763594.300944:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c44e0.
00000100:00000001:1.0:1787763594.300952:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787763594.300960:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315982579
00000100:00000010:1.0:1787763594.300965:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c49c0.
00000100:00000001:1.0:1787763594.300969:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763594.300973:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763594.300976:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763594.300978:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763594.300985:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763594/real 1787763594]  req@ffff9751f865ad00 x1874599832640384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763594.300997:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763594.300999:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763594.301002:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763594.301005:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763594.301008:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763594.301011:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763594.301017:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832640384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763594.301044:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832640384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301056:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763594.301058:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.301062:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763594.301076:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763594.301082:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832640384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301094:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763594.301100:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832640384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301123:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763594.301127:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.301129:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763594.301131:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.301151:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763594.301152:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763594.301155:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763594.301158:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763594.301160:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763594.301164:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832640384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763594.301180:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832640384:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763594.301189:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.301192:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763594.301197:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832640384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301206:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763594.301209:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763594.301212:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763594.301218:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763594.301220:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763594.301224:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763594.301226:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763594.301242:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:1.0:1787763594.301246:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763594.301249:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763594.301251:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:1.0:1787763594.301255:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763594.301258:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.301262:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.301269:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763594.301272:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763594.301277:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763594.301278:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763594.301279:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763594.301283:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832640512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301294:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763594.301296:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763594.301299:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763594.301302:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763594.301305:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763594.301310:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832640512:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763594.301325:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763594.301326:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763594.301329:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763594.301331:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763594.301334:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763594.301339:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763594.301343:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763594.301348:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763594.301351:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763594.301355:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832640512, portal 10
00000100:00000001:1.0:1787763594.301359:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763594.301360:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787763594.301369:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832640512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763619 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301389:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763594.301395:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763594.301398:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763594.301401:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832640512, offset 0
00000400:00000200:1.0:1787763594.301408:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763594.301414:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763594.301421:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763594.301424:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763594.301429:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763594.301432:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763594.301436:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763594.301440:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58239:58239:256:4294967295] 192.168.203.110@tcp LPNI seq info [58239:58239:8:4294967295]
00000400:00000200:1.0:1787763594.301443:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763594.301448:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763594.301453:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763594.301455:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763594.301458:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763594.301460:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787763594.301463:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763594.301465:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1787763594.301467:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763594.301468:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763594.301470:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763594.301473:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763594.301475:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763594.301476:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763594.301478:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763594.301480:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832640512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763619 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301485:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763594.301487:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832640512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763619 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301492:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763594.301493:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763594.301495:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763594.301497:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763594.301500:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763594.301501:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315981344
00000100:00000001:1.0:1787763594.301503:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315982579
00000100:00000001:1.0:1787763594.301505:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315981344
00000100:00000001:1.0:1787763594.301507:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315982579
00000100:00000001:1.0:1787763594.301508:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763594.301509:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763594.301510:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763594.301513:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763594/real 1787763594]  req@ffff9751f8658000 x1874599832640512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763594.301517:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763594.301518:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763594.301519:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763594.301521:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763594.301522:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763594.301523:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763594.301525:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832640512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763594.301530:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832640512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301534:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763594.301534:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.301536:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763594.301536:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763594.301538:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832640512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301542:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763594.301545:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832640512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301549:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763594.301550:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.301552:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763594.301553:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.301554:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763594.301564:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763594.301566:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763594.301568:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763594.301570:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763594.301572:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832640512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763594.301577:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832640512:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763594.301579:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763594.301581:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832640640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301585:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763594.301585:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763594.301587:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763594.301588:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763594.301589:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763594.301591:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832640640:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763594.301593:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763594.301594:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763594.301595:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763594.301596:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763594.301599:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763594.301600:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763594.301602:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763594.301603:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763594.301605:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763594.301607:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832640640, portal 10
00000100:00000001:1.0:1787763594.301608:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763594.301609:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787763594.301614:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832640640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763619 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301621:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763594.301624:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763594.301626:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763594.301629:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832640640, offset 0
00000400:00000200:1.0:1787763594.301633:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763594.301637:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763594.301640:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763594.301643:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763594.301645:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763594.301648:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763594.301650:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763594.301653:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58240:58240:256:4294967295] 192.168.203.110@tcp LPNI seq info [58240:58240:8:4294967295]
00000400:00000200:1.0:1787763594.301657:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763594.301663:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763594.301667:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763594.301670:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763594.301674:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763594.301677:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787763594.301682:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763594.301684:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1787763594.301688:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763594.301690:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763594.301693:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763594.301696:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763594.301699:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763594.301701:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763594.301704:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763594.301707:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832640640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763619 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301712:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763594.301715:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832640640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763619 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301730:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763594.301731:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763594.301734:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763594.301736:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763594.301738:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763594.301739:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315981332
00000100:00000001:1.0:1787763594.301741:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315982579
00000100:00000001:1.0:1787763594.301743:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315981332
00000100:00000001:1.0:1787763594.301744:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315982579
00000100:00000001:1.0:1787763594.301746:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763594.301747:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763594.301749:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763594.301751:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763594/real 1787763594]  req@ffff9751f8659a40 x1874599832640640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763594.301756:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763594.301757:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763594.301758:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763594.301759:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763594.301761:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763594.301762:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763594.301764:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832640640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763594.301769:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832640640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301772:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763594.301773:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.301774:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763594.301775:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763594.301778:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832640640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301793:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763594.301795:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832640640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301799:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763594.301801:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.301802:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763594.301802:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.301803:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763594.301804:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763594.301806:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763594.301808:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763594.301809:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763594.301811:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832640640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763594.301816:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832640640:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763594.301818:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.301819:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763594.301821:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832640512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301825:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763594.301827:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763594.301828:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763594.301830:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763594.301831:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763594.301832:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763594.301833:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763594.301835:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:1.0:1787763594.301836:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763594.301837:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763594.301838:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:1.0:1787763594.301840:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763594.301841:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.301842:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763594.301844:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832640640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301848:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763594.301849:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763594.301849:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763594.301851:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763594.301851:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763594.301852:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763594.301853:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763594.301854:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:1.0:1787763594.301855:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763594.301856:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763594.301856:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787763594.301858:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763594.301859:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.301860:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.301862:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763594.301863:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763594.301864:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763594.301865:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763594.301866:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763594.301868:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832640768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301872:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763594.301873:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763594.301874:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763594.301875:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763594.301876:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763594.301878:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832640768:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763594.301881:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763594.301882:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763594.301884:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763594.301885:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763594.301887:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763594.301888:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763594.301890:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763594.301892:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763594.301894:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763594.301897:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832640768, portal 4
00000100:00000001:1.0:1787763594.301898:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763594.301899:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000040:1.0:1787763594.301903:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832640768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763619 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301907:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763594.301908:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763594.301910:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763594.301911:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832640768, offset 0
00000400:00000200:1.0:1787763594.301914:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763594.301916:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763594.301918:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763594.301920:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763594.301922:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763594.301924:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763594.301926:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763594.301928:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58241:58241:256:4294967295] 192.168.203.110@tcp LPNI seq info [58241:58241:8:4294967295]
00000400:00000200:1.0:1787763594.301931:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763594.301935:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763594.301938:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763594.301939:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763594.301942:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763594.301944:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787763594.301947:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763594.301949:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1787763594.301951:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763594.301952:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763594.301954:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763594.301956:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763594.301958:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763594.301959:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763594.301961:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763594.301963:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832640768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763619 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301968:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763594.301969:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832640768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763619 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.301985:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763594.301986:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763594.301987:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763594.301989:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763594.301991:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763594.301992:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315981349
00000100:00000001:1.0:1787763594.301994:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315982579
00000100:00000001:1.0:1787763594.301995:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315981349
00000100:00000001:1.0:1787763594.301997:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315982579
00000100:00000001:1.0:1787763594.301998:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763594.301999:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763594.302000:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763594.302002:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763594/real 1787763594]  req@ffff9751f8658780 x1874599832640768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763594.302007:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763594.302008:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763594.302009:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763594.302011:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763594.302012:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763594.302013:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763594.302015:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832640768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763594.302020:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832640768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.302039:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763594.302040:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.302042:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763594.302043:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763594.302045:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832640768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.302049:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763594.302052:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832640768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.302057:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763594.302059:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.302060:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763594.302060:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.302062:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763594.302063:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763594.302064:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763594.302066:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763594.302067:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763594.302070:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832640768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763594.302074:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832640768:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763594.302077:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.302078:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763594.302080:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832640768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763619 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763594.302084:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763594.302085:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763594.302086:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763594.302088:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763594.302088:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763594.302089:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763594.302090:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763594.302091:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:1.0:1787763594.302093:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763594.302094:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763594.302094:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:1.0:1787763594.302096:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763594.302097:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.302098:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763594.302100:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763594.302101:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763594.302102:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763594.302103:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763595.964308:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763595.964314:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763595.964318:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763596.988300:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763596.988305:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763596.988308:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763599.036304:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763599.036308:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763599.036311:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763599.420615:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763599.420623:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763599.420628:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763599.420639:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832640896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.420660:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763599.420665:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763599.420671:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763599.420677:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763599.420681:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763599.420689:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832640896:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763599.420698:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763599.420703:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763599.420708:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763599.420712:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763599.420719:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763599.420725:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763599.420736:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763599.420741:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763599.420749:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763599.420759:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832640896, portal 25
00000100:00000001:1.0:1787763599.420763:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763599.420766:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:1.0:1787763599.420778:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832640896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763624 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.420811:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763599.420818:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763599.420823:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763599.420831:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832640896, offset 0
00000400:00000200:1.0:1787763599.420848:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763599.420862:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763599.420874:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763599.420886:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763599.420896:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763599.420904:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763599.420911:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763599.420923:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58242:58242:256:4294967295] 192.168.203.110@tcp LPNI seq info [58242:58242:8:4294967295]
00000400:00000200:1.0:1787763599.420935:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763599.420961:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763599.420974:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763599.420982:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763599.421012:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763599.421021:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787763599.421032:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763599.421038:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787763599.421047:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763599.421054:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763599.421063:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763599.421071:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763599.421078:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763599.421082:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763599.421090:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763599.421101:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832640896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763624 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.421128:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763599.421172:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832640896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763624 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.421193:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763599.421204:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763599.421217:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763599.421225:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763599.421232:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315987700
00000100:00000010:1.0:1787763599.421242:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4b40.
00000100:00000001:1.0:1787763599.421254:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787763599.421268:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315987700
00000100:00000010:1.0:1787763599.421276:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4f60.
00000100:00000001:1.0:1787763599.421282:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763599.421289:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763599.421297:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763599.421303:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763599.421312:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763599/real 1787763599]  req@ffff9751f86583c0 x1874599832640896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763599.421334:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763599.421338:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763599.421343:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763599.421350:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763599.421355:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763599.421361:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763599.421371:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832640896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763599.421396:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832640896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.421415:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763599.421419:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763599.421424:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763599.421427:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763599.421439:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832640896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.421466:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763599.421476:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832640896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.421486:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763599.421490:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763599.421492:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763599.421493:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763599.421496:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763599.421499:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763599.421502:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763599.421507:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763599.421510:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763599.421514:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832640896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763599.421523:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832640896:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763599.421528:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763599.421531:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763599.421534:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832640896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.421547:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763599.421549:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763599.421552:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763599.421556:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763599.421558:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763599.421560:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763599.421562:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763599.421565:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:1.0:1787763599.421569:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763599.421571:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763599.421572:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:1.0:1787763599.421575:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763599.421577:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763599.421579:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763599.421586:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763599.421588:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763599.421591:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763599.421592:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763599.421594:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763599.421598:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832641024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.421606:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763599.421608:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763599.421610:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763599.421612:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763599.421615:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763599.421618:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832641024:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763599.421622:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763599.421624:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763599.421626:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763599.421628:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763599.421632:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763599.421634:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763599.421639:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763599.421642:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763599.421646:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763599.421650:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832641024, portal 10
00000100:00000001:1.0:1787763599.421652:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763599.421654:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000040:1.0:1787763599.421661:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832641024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763624 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.421668:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763599.421671:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763599.421673:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763599.421676:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832641024, offset 0
00000400:00000200:1.0:1787763599.421680:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763599.421685:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763599.421689:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763599.421693:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763599.421696:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763599.421699:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763599.421702:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763599.421705:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58243:58243:256:4294967295] 192.168.203.110@tcp LPNI seq info [58243:58243:8:4294967295]
00000400:00000200:1.0:1787763599.421709:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763599.421716:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763599.421721:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763599.421725:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763599.421730:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763599.421733:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787763599.421738:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763599.421741:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787763599.421745:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763599.421747:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763599.421751:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763599.421754:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763599.421757:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763599.421760:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763599.421763:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763599.421767:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832641024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763624 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.421775:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763599.421779:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832641024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763624 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.421786:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763599.421788:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763599.421791:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763599.421795:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763599.421798:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763599.421801:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315986464
00000100:00000001:1.0:1787763599.421804:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315987700
00000100:00000001:1.0:1787763599.421806:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315986464
00000100:00000001:1.0:1787763599.421808:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315987700
00000100:00000001:1.0:1787763599.421810:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763599.421813:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763599.421815:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763599.421819:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763599/real 1787763599]  req@ffff9751f8658780 x1874599832641024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763599.421827:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763599.421828:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763599.421830:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763599.421833:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763599.421835:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763599.421837:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763599.421841:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832641024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763599.421849:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832641024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.421862:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763599.421864:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763599.421865:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763599.421867:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763599.421871:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832641024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.421878:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763599.421882:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832641024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.421889:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763599.421892:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763599.421894:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763599.421895:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763599.421904:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763599.421917:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763599.421920:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763599.421924:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763599.421926:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763599.421930:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832641024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763599.421939:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832641024:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763599.421943:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763599.421946:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832641152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.421953:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763599.421954:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763599.421957:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763599.421959:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763599.421961:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763599.421964:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832641152:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763599.421967:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763599.421969:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763599.421970:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763599.421972:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763599.421976:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763599.421978:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763599.421982:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763599.421984:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763599.421988:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763599.422005:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832641152, portal 10
00000100:00000001:1.0:1787763599.422008:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763599.422009:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787763599.422015:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832641152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763624 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.422022:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763599.422024:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763599.422027:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763599.422030:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832641152, offset 0
00000400:00000200:1.0:1787763599.422034:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763599.422039:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763599.422043:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763599.422046:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763599.422049:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763599.422052:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763599.422054:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763599.422057:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58244:58244:256:4294967295] 192.168.203.110@tcp LPNI seq info [58244:58244:8:4294967295]
00000400:00000200:1.0:1787763599.422061:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763599.422068:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763599.422073:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763599.422077:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763599.422081:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763599.422084:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787763599.422089:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763599.422092:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787763599.422096:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763599.422097:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763599.422101:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763599.422104:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763599.422108:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763599.422110:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763599.422112:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763599.422116:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832641152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763624 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.422124:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763599.422127:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832641152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763624 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.422166:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763599.422169:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763599.422174:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763599.422177:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763599.422181:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763599.422183:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315986453
00000100:00000001:1.0:1787763599.422186:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315987700
00000100:00000001:1.0:1787763599.422188:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315986453
00000100:00000001:1.0:1787763599.422190:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315987700
00000100:00000001:1.0:1787763599.422193:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763599.422195:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763599.422197:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763599.422201:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763599/real 1787763599]  req@ffff9751f8659a40 x1874599832641152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763599.422210:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763599.422212:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763599.422214:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763599.422216:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763599.422218:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763599.422220:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763599.422223:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832641152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763599.422232:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832641152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.422239:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763599.422241:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763599.422242:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763599.422244:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763599.422247:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832641152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.422254:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763599.422259:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832641152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.422266:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763599.422269:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763599.422270:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763599.422271:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763599.422274:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763599.422275:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763599.422279:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763599.422282:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763599.422284:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763599.422289:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832641152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763599.422298:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832641152:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763599.422302:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763599.422306:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832641280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.422313:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763599.422315:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763599.422318:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763599.422320:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763599.422322:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763599.422325:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832641280:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763599.422328:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763599.422330:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763599.422332:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763599.422334:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763599.422338:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763599.422340:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763599.422344:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763599.422346:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763599.422349:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763599.422353:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832641280, portal 4
00000100:00000001:1.0:1787763599.422355:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763599.422356:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787763599.422362:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832641280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763624 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.422369:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763599.422371:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763599.422374:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763599.422376:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832641280, offset 0
00000400:00000200:1.0:1787763599.422381:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763599.422386:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763599.422390:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763599.422393:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763599.422396:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763599.422399:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763599.422402:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763599.422405:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58245:58245:256:4294967295] 192.168.203.110@tcp LPNI seq info [58245:58245:8:4294967295]
00000400:00000200:1.0:1787763599.422409:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763599.422416:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763599.422421:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763599.422425:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763599.422429:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763599.422433:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787763599.422437:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763599.422441:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787763599.422445:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763599.422447:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763599.422450:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763599.422454:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763599.422457:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763599.422459:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763599.422462:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763599.422466:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832641280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763624 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.422475:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763599.422478:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832641280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763624 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.422485:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763599.422487:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763599.422490:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763599.422493:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763599.422497:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763599.422499:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315986470
00000100:00000001:1.0:1787763599.422502:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315987700
00000100:00000001:1.0:1787763599.422504:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315986470
00000100:00000001:1.0:1787763599.422506:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315987700
00000100:00000001:1.0:1787763599.422509:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763599.422511:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763599.422513:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763599.422517:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763599/real 1787763599]  req@ffff9751f8658000 x1874599832641280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763599.422526:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763599.422527:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763599.422529:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763599.422531:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763599.422533:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763599.422536:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763599.422539:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832641280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763599.422548:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832641280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.422555:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763599.422557:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763599.422559:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763599.422560:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763599.422564:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832641280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.422571:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763599.422575:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832641280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.422582:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763599.422585:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763599.422587:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763599.422589:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763599.422591:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763599.422592:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763599.422595:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763599.422599:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763599.422601:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763599.422605:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832641280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763599.422613:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832641280:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763599.422617:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763599.422619:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763599.422622:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832641024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.422629:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763599.422631:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763599.422633:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763599.422636:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763599.422638:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763599.422639:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763599.422642:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763599.422644:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:1.0:1787763599.422646:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763599.422648:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763599.422649:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:1.0:1787763599.422652:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763599.422653:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763599.422655:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763599.422658:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832641152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.422671:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763599.422672:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763599.422674:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763599.422676:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763599.422677:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763599.422678:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763599.422680:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763599.422682:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:1.0:1787763599.422684:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763599.422685:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763599.422687:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787763599.422689:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763599.422690:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763599.422692:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763599.422695:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832641280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763624 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763599.422702:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763599.422703:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763599.422705:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763599.422707:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763599.422709:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763599.422710:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763599.422712:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763599.422714:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:1.0:1787763599.422716:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763599.422717:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763599.422719:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:1.0:1787763599.422721:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763599.422722:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763599.422724:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763599.422729:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763599.422730:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763599.422732:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763599.422734:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763600.060360:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763600.060364:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763600.060366:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763601.084275:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763601.084279:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763601.084282:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763602.108331:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763602.108335:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763602.108338:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763603.132330:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763603.132334:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763603.132337:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763603.516276:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763603.516284:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763603.516286:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763603.516291:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763603.516295:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763603.516299:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763603.516302:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763603.516305:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763603.516309:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763604.156369:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763604.156375:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763604.156379:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763604.540241:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763604.540249:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763604.540253:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763604.540260:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763604.540266:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763604.540271:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763604.540276:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763604.540280:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763604.540285:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763604.540444:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763604.540450:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763604.540454:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763604.540467:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832641408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.540489:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763604.540493:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763604.540501:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763604.540504:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763604.540508:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763604.540516:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832641408:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763604.540525:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763604.540533:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763604.540538:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763604.540545:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763604.540555:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763604.540561:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763604.540573:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763604.540582:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763604.540596:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763604.540607:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832641408, portal 25
00000100:00000001:1.0:1787763604.540614:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763604.540618:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:1.0:1787763604.540633:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832641408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763629 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.540651:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763604.540657:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763604.540662:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763604.540668:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832641408, offset 0
00000400:00000200:1.0:1787763604.540680:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763604.540698:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763604.540707:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763604.540717:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763604.540722:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763604.540729:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763604.540736:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763604.540744:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58246:58246:256:4294967295] 192.168.203.110@tcp LPNI seq info [58246:58246:8:4294967295]
00000400:00000200:1.0:1787763604.540756:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763604.540770:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763604.540782:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763604.540790:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763604.540801:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763604.540808:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787763604.540822:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763604.540827:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787763604.540837:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763604.540841:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763604.540847:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763604.540856:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763604.540861:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763604.540864:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763604.540870:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763604.540879:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832641408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763629 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.540897:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763604.540902:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832641408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763629 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.540919:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763604.540924:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763604.540931:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763604.540939:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763604.540943:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315992819
00000100:00000010:1.0:1787763604.540951:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4e40.
00000100:00000001:1.0:1787763604.540978:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787763604.540987:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315992819
00000100:00000010:1.0:1787763604.540994:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4060.
00000100:00000001:1.0:1787763604.540997:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763604.541001:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763604.541004:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763604.541006:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763604.541011:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763604/real 1787763604]  req@ffff9751f865ad00 x1874599832641408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763604.541018:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763604.541020:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763604.541021:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763604.541023:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763604.541024:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763604.541026:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763604.541028:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832641408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763604.541034:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832641408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541038:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763604.541039:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763604.541041:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763604.541042:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763604.541045:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832641408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541050:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763604.541053:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832641408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541058:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763604.541060:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763604.541061:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763604.541061:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763604.541063:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763604.541065:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763604.541067:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763604.541070:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763604.541071:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763604.541074:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832641408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763604.541079:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832641408:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763604.541082:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763604.541084:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763604.541086:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832641408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541090:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763604.541091:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763604.541093:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763604.541095:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763604.541097:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763604.541098:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763604.541099:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763604.541101:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:1.0:1787763604.541104:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763604.541106:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763604.541107:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:1.0:1787763604.541109:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763604.541110:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763604.541111:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763604.541117:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763604.541117:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763604.541119:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763604.541120:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763604.541121:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763604.541125:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832641536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541160:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763604.541162:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763604.541164:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763604.541166:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763604.541169:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763604.541173:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832641536:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763604.541177:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763604.541179:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763604.541181:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763604.541183:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763604.541186:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763604.541187:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763604.541190:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763604.541192:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763604.541194:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763604.541197:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832641536, portal 10
00000100:00000001:1.0:1787763604.541198:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763604.541199:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787763604.541204:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832641536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763629 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541208:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763604.541210:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763604.541211:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763604.541213:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832641536, offset 0
00000400:00000200:1.0:1787763604.541216:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763604.541219:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763604.541222:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763604.541224:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763604.541226:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763604.541228:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763604.541229:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763604.541232:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58247:58247:256:4294967295] 192.168.203.110@tcp LPNI seq info [58247:58247:8:4294967295]
00000400:00000200:1.0:1787763604.541235:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763604.541239:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763604.541243:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763604.541245:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763604.541248:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763604.541250:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787763604.541253:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763604.541254:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787763604.541257:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763604.541258:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763604.541260:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763604.541263:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763604.541265:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763604.541266:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763604.541268:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763604.541270:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832641536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763629 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541275:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763604.541277:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832641536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763629 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541282:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763604.541283:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763604.541285:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763604.541288:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763604.541290:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763604.541291:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315991584
00000100:00000001:1.0:1787763604.541293:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315992819
00000100:00000001:1.0:1787763604.541295:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315991584
00000100:00000001:1.0:1787763604.541296:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315992819
00000100:00000001:1.0:1787763604.541298:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763604.541299:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763604.541301:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763604.541303:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763604/real 1787763604]  req@ffff9751f8658000 x1874599832641536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763604.541308:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763604.541309:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763604.541310:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763604.541312:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763604.541313:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763604.541314:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763604.541316:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832641536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763604.541321:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832641536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541325:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763604.541326:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763604.541327:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763604.541328:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763604.541330:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832641536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541334:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763604.541336:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832641536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541340:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763604.541342:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763604.541343:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763604.541344:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763604.541345:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763604.541354:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763604.541355:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763604.541358:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763604.541359:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763604.541362:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832641536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763604.541367:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832641536:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763604.541369:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763604.541371:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832641664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541375:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763604.541376:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763604.541377:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763604.541378:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763604.541379:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763604.541381:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832641664:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763604.541383:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763604.541385:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763604.541386:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763604.541387:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763604.541390:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763604.541392:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763604.541395:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763604.541397:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763604.541400:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763604.541403:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832641664, portal 10
00000100:00000001:1.0:1787763604.541405:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763604.541406:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787763604.541412:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832641664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763629 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541419:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763604.541422:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763604.541424:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763604.541427:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832641664, offset 0
00000400:00000200:1.0:1787763604.541431:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763604.541435:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763604.541439:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763604.541442:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763604.541445:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763604.541447:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763604.541449:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763604.541451:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58248:58248:256:4294967295] 192.168.203.110@tcp LPNI seq info [58248:58248:8:4294967295]
00000400:00000200:1.0:1787763604.541454:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763604.541459:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763604.541464:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763604.541467:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763604.541471:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763604.541474:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787763604.541478:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763604.541481:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787763604.541485:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763604.541487:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763604.541490:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763604.541494:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763604.541497:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763604.541500:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763604.541503:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763604.541507:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832641664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763629 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541515:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763604.541518:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832641664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763629 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541525:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763604.541527:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763604.541532:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763604.541535:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763604.541539:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763604.541541:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315991572
00000100:00000001:1.0:1787763604.541543:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315992819
00000100:00000001:1.0:1787763604.541546:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315991572
00000100:00000001:1.0:1787763604.541548:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315992819
00000100:00000001:1.0:1787763604.541551:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763604.541553:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763604.541555:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763604.541559:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763604/real 1787763604]  req@ffff9751f8659a40 x1874599832641664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763604.541569:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763604.541570:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763604.541571:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763604.541573:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763604.541574:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763604.541576:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763604.541578:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832641664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763604.541587:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832641664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541594:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763604.541596:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763604.541597:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763604.541599:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763604.541603:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832641664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541610:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763604.541615:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832641664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541623:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763604.541626:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763604.541627:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763604.541629:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763604.541631:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763604.541633:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763604.541635:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763604.541638:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763604.541640:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763604.541643:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832641664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763604.541648:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832641664:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763604.541650:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763604.541652:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832641792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541656:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763604.541657:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763604.541659:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763604.541660:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763604.541662:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763604.541664:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832641792:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763604.541666:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763604.541667:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763604.541668:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763604.541669:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763604.541672:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763604.541673:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763604.541676:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763604.541678:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763604.541680:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763604.541682:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832641792, portal 4
00000100:00000001:1.0:1787763604.541684:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763604.541685:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000040:1.0:1787763604.541688:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832641792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763629 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541693:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763604.541694:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763604.541696:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763604.541697:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832641792, offset 0
00000400:00000200:1.0:1787763604.541700:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763604.541703:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763604.541705:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763604.541707:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763604.541709:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763604.541711:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763604.541713:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763604.541715:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58249:58249:256:4294967295] 192.168.203.110@tcp LPNI seq info [58249:58249:8:4294967295]
00000400:00000200:1.0:1787763604.541718:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763604.541722:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763604.541725:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763604.541727:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763604.541730:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763604.541732:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787763604.541734:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763604.541736:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787763604.541739:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763604.541740:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763604.541742:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763604.541745:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763604.541747:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763604.541748:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763604.541750:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763604.541752:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832641792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763629 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541757:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763604.541759:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832641792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763629 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541764:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763604.541765:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763604.541766:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763604.541768:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763604.541770:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763604.541772:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315991589
00000100:00000001:1.0:1787763604.541773:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315992819
00000100:00000001:1.0:1787763604.541775:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315991589
00000100:00000001:1.0:1787763604.541776:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315992819
00000100:00000001:1.0:1787763604.541778:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763604.541779:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763604.541780:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763604.541782:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763604/real 1787763604]  req@ffff9751f8658780 x1874599832641792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763604.541787:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763604.541788:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763604.541789:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763604.541790:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763604.541791:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763604.541793:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763604.541795:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832641792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763604.541800:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832641792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541805:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763604.541806:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763604.541807:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763604.541808:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763604.541810:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832641792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541814:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763604.541816:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832641792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541820:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763604.541822:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763604.541823:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763604.541823:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763604.541824:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763604.541825:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763604.541827:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763604.541828:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763604.541830:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763604.541832:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832641792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763604.541836:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832641792:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763604.541839:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763604.541840:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763604.541842:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832641536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541846:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763604.541847:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763604.541849:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763604.541850:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763604.541851:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763604.541853:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763604.541854:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763604.541855:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:1.0:1787763604.541857:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763604.541858:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763604.541859:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:1.0:1787763604.541861:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763604.541862:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763604.541863:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763604.541865:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832641664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541869:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763604.541870:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763604.541871:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763604.541872:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763604.541872:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763604.541873:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763604.541874:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763604.541875:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:1.0:1787763604.541876:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763604.541877:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763604.541878:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787763604.541879:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763604.541880:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763604.541881:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763604.541882:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832641792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763629 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763604.541886:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763604.541887:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763604.541888:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763604.541889:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763604.541889:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763604.541890:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763604.541892:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763604.541892:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:1.0:1787763604.541894:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763604.541895:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763604.541895:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:1.0:1787763604.541897:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763604.541897:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763604.541898:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763604.541901:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763604.541902:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763604.541903:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763604.541904:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763605.180348:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763605.180353:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763605.180358:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763605.564225:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763605.564231:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763605.564233:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763605.564238:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763605.564241:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763605.564244:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763605.564247:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763605.564249:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763605.564251:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763606.204360:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763606.204365:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763606.204369:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763606.588215:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763606.588221:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763606.588223:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763606.588228:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763606.588231:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763606.588234:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763606.588236:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763606.588238:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763606.588240:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763607.228385:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763607.228388:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763607.228391:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763607.612250:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763607.612259:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763607.612261:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763607.612267:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763607.612277:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763607.612280:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763607.612283:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763607.612286:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763607.612288:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763608.252365:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763608.252369:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763608.252372:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763608.636222:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763608.636231:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763608.636234:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763608.636239:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763608.636243:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763608.636247:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763608.636250:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763608.636253:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763608.636256:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763609.276314:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763609.276319:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763609.276323:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763609.660243:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763609.660251:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763609.660254:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763609.660257:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763609.660262:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763609.660270:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21324
00000400:00000200:0.0:1787763609.660280:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763609.660288:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763609.660292:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763609.660296:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763609.660299:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763609.660301:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763609.660303:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763609.660306:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763609.660308:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763609.660310:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763609.660313:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763609.660314:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763609.660316:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763609.660319:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763609.660320:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763609.660323:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763609.660324:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763609.660327:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763609.660329:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763609.660331:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763609.660333:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763609.660336:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763609.660338:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763609.660340:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763609.660342:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763609.660350:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
00000020:00000040:0.0:1787763609.660352:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763609.660355:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763609.660356:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763609.660358:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763609.660360:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763609.660364:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763609.660371:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763609.660376:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763609.660384:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832641920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763609.660392:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763609.660395:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763609.660398:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832641920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763609.660413:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763609.660415:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763609.660417:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763609.660418:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763609.660419:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763609.660421:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763609.660425:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21324
00000400:00000200:0.0:1787763609.660429:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763609.660432:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763609.660435:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763609.660437:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763609.660439:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763609.660442:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763609.660442:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763609.660445:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787763609.660448:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1787763609.660451:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:1.0:1787763609.660452:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1787763609.660455:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763609.660459:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763609.660463:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000040:1.0:1787763609.660465:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876da40 x1874599832641920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787763609.660467:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763609.660473:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763609.660483:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763609.660485:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763609.660489:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787763609.660489:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763609.660495:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:1.0:1787763609.660496:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1787763609.660500:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:1.0:1787763609.660501:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:0.0:1787763609.660503:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:1.0:1787763609.660506:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763609.660508:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763609.660512:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763609.660521:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832641920:192.168.203.110@tcp:250:kworker.0
02000000:00000001:0.0:1787763609.660525:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787763609.660532:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763609.660533:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763609.660536:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763609.660539:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763609.660541:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763609.660544:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1787763609.660546:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:0.0:1787763609.660551:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000010:1.0:1787763609.660553:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
00000008:00000001:0.0:1787763609.660556:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763609.660558:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763609.660559:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763609.660566:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000400:00000200:1.0:1787763609.660568:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:0.0:1787763609.660570:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763609.660574:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:1.0:1787763609.660575:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
02000000:00000001:0.0:1787763609.660576:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763609.660580:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763609.660581:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763609.660583:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763609.660584:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
02000000:00000010:0.0:1787763609.660588:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000200:1.0:1787763609.660592:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832641920, portal 25
00000100:00000001:0.0:1787763609.660593:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763609.660598:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763609.660600:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049524800 : -115096660026816 : ffff9751f876da40)
00000100:00100000:0.0:1787763609.660602:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832642048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763609.660612:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:1.0:1787763609.660613:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876da40 x1874599832641920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763634 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763609.660614:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763609.660626:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763609.660628:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832642048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787763609.660633:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763609.660638:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000001:0.0:1787763609.660642:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787763609.660644:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832641920, offset 0
00000100:00080000:0.0:1787763609.660646:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763609.660651:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763609.660654:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1787763609.660655:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00080000:0.0:1787763609.660656:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763609.660661:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763609.660664:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21324
00000400:00000200:1.0:1787763609.660668:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763609.660675:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763609.660681:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763609.660685:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763609.660689:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763609.660692:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763609.660697:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58250:58250:256:4294967295] 192.168.203.110@tcp LPNI seq info [58250:58250:8:4294967295]
00000400:00000200:1.0:1787763609.660701:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763609.660707:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763609.660714:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763609.660718:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1787763609.660719:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:0.0:1787763609.660725:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763609.660729:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.0:1787763609.660733:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787763609.660734:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763609.660739:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763609.660740:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763609.660741:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000800:00000010:1.0:1787763609.660742:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000020:00000001:0.0:1787763609.660743:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763609.660747:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763609.660750:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763609.660755:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:1.0:1787763609.660755:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763609.660763:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:1.0:1787763609.660764:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:0.0:1787763609.660765:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763609.660770:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763609.660772:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763609.660776:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000800:00000200:1.1:1787763609.660777:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00080000:0.0:1787763609.660780:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763609.660784:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787763609.660785:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
02000000:00000001:0.0:1787763609.660786:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763609.660790:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763609.660794:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:1.0:1787763609.660795:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763609.660796:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000010:1.0:1787763609.660800:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000008:00100000:0.0:1787763609.660801:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763609.660806:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763609.660807:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763609.660810:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000010:0.0:1787763609.660811:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000020:00000040:0.0:1787763609.660815:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000400:00000200:1.0:1787763609.660818:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:0.0:1787763609.660819:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763609.660823:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763609.660825:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:1.0:1787763609.660826:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
02000000:00000001:0.0:1787763609.660827:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763609.660831:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763609.660832:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
02000000:00000010:0.0:1787763609.660834:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:1.0:1787763609.660838:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1787763609.660843:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787763609.660844:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876da40 x1874599832641920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763634 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763609.660851:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832642176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763609.660869:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1787763609.660871:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763609.660877:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763609.660878:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876da40 x1874599832641920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763634 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763609.660888:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832642176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.660892:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763609.660896:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763609.660903:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763609.660905:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763609.660911:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:0.0:1787763609.660912:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787763609.660915:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4315997939
00000100:00000040:0.0:1787763609.660917:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763609.660934:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763609.660938:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000010:1.0:1787763609.660938:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4720.
00000100:00000001:0.0:1787763609.660939:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763609.660943:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763609.660946:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763609.660949:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21324
00000100:00000001:1.0:1787763609.660949:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000400:00000200:0.0:1787763609.660956:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763609.660960:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1787763609.660960:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4315997939
00000100:00080000:0.0:1787763609.660964:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000010:1.0:1787763609.660967:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c45a0.
00000100:00000040:0.0:1787763609.660968:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763609.660972:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763609.660974:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763609.660974:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787763609.660979:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787763609.660980:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000001:0.0:1787763609.660981:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763609.660984:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000100:00000001:1.0:1787763609.660986:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763609.660987:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763609.660991:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000001:1.0:1787763609.660992:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000020:00000001:0.0:1787763609.660993:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763609.660997:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00100000:1.0:1787763609.661001:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763609/real 1787763609]  req@ffff9751f876da40 x1874599832641920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763609.661002:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763609.661007:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763609.661011:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763609.661014:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763609.661019:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763609.661024:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763609.661028:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1787763609.661029:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:0.0:1787763609.661030:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000001:1.0:1787763609.661032:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:0.0:1787763609.661035:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:1.0:1787763609.661036:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
02000000:00000001:0.0:1787763609.661040:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763609.661042:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000100:00000001:0.0:1787763609.661043:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763609.661048:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000400:00000010:1.0:1787763609.661048:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000008:00000001:0.0:1787763609.661054:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763609.661055:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:0.0:1787763609.661056:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763609.661062:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000100:00000200:1.0:1787763609.661062:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876da40 x1874599832641920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787763609.661067:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763609.661072:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763609.661075:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763609.661079:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763609.661082:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763609.661085:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787763609.661086:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876da40 x1874599832641920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1787763609.661089:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763609.661094:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763609.661102:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00100000:0.0:1787763609.661103:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832642304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763609.661105:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763609.661112:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763609.661114:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763609.661116:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763609.661118:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763609.661129:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832642304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763609.661130:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876da40 x1874599832641920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.661165:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763609.661168:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763609.661174:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21334)
00000100:00000040:1.0:1787763609.661175:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876da40 x1874599832641920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.661184:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763609.661188:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763609.661191:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763609.661193:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763609.661195:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763609.661197:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763609.661200:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763609.661205:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763609.661208:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763609.661213:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876da40 x1874599832641920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763609.661222:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832641920:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763609.661227:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763609.661229:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763609.661232:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876da40 x1874599832641920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.661239:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763609.661241:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763609.661244:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763609.661247:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763609.661249:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763609.661251:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763609.661254:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763609.661256:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017800.
02000000:00000001:1.0:1787763609.661261:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763609.661263:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763609.661265:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876da40.
00000100:00000001:1.0:1787763609.661268:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763609.661270:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763609.661272:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763609.661278:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763609.661280:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763609.661282:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763609.661284:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763609.661285:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763609.661289:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876de00 x1874599832642048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.661297:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763609.661299:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763609.661301:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763609.661303:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763609.661306:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763609.661309:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832642048:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763609.661312:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763609.661314:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763609.661316:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763609.661318:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763609.661321:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763609.661323:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763609.661327:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763609.661329:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763609.661333:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763609.661336:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832642048, portal 10
00000100:00000001:1.0:1787763609.661338:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763609.661339:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049525760 : -115096660025856 : ffff9751f876de00)
00000100:00000040:1.0:1787763609.661346:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876de00 x1874599832642048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763634 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.661353:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763609.661355:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763609.661358:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763609.661361:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832642048, offset 0
00000400:00000200:1.0:1787763609.661365:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763609.661369:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763609.661373:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763609.661376:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763609.661379:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763609.661382:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763609.661384:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763609.661388:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58251:58251:256:4294967295] 192.168.203.110@tcp LPNI seq info [58251:58251:8:4294967295]
00000400:00000200:1.0:1787763609.661393:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763609.661399:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763609.661404:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763609.661408:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763609.661412:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763609.661415:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787763609.661420:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763609.661423:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787763609.661427:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763609.661429:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763609.661432:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763609.661436:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763609.661440:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763609.661442:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763609.661444:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763609.661448:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876de00 x1874599832642048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763634 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.661457:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763609.661461:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876de00 x1874599832642048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763634 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.661468:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763609.661469:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763609.661473:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763609.661477:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763609.661480:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763609.661482:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315996704
00000100:00000001:1.0:1787763609.661485:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315997939
00000100:00000001:1.0:1787763609.661488:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315996704
00000100:00000001:1.0:1787763609.661490:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315997939
00000100:00000001:1.0:1787763609.661493:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763609.661495:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763609.661497:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763609.661501:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763609/real 1787763609]  req@ffff9751f876de00 x1874599832642048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763609.661509:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763609.661511:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763609.661513:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763609.661515:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763609.661517:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763609.661519:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763609.661523:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876de00 x1874599832642048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763609.661532:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876de00 x1874599832642048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.661539:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763609.661541:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763609.661543:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763609.661544:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763609.661548:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876de00 x1874599832642048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.661554:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763609.661559:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876de00 x1874599832642048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.661567:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763609.661569:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763609.661571:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763609.661573:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763609.661575:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763609.661587:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763609.661590:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763609.661593:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763609.661596:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763609.661600:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876de00 x1874599832642048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763609.661609:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832642048:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763609.661613:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763609.661617:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cb40 x1874599832642176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.661624:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763609.661626:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763609.661628:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763609.661630:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763609.661632:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763609.661635:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832642176:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763609.661638:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763609.661640:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763609.661642:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763609.661644:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763609.661647:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763609.661649:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763609.661652:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763609.661655:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763609.661658:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763609.661661:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832642176, portal 10
00000100:00000001:1.0:1787763609.661663:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763609.661664:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049520960 : -115096660030656 : ffff9751f876cb40)
00000100:00000040:1.0:1787763609.661671:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cb40 x1874599832642176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763634 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.661678:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763609.661680:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763609.661683:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763609.661685:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832642176, offset 0
00000400:00000200:1.0:1787763609.661689:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763609.661693:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763609.661696:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763609.661699:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763609.661702:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763609.661704:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763609.661707:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763609.661711:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58252:58252:256:4294967295] 192.168.203.110@tcp LPNI seq info [58252:58252:8:4294967295]
00000400:00000200:1.0:1787763609.661715:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763609.661722:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763609.661727:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763609.661730:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763609.661734:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763609.661737:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787763609.661742:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763609.661745:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787763609.661749:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763609.661751:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763609.661754:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763609.661758:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763609.661761:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763609.661763:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763609.661765:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763609.661769:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cb40 x1874599832642176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763634 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.661778:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763609.661781:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cb40 x1874599832642176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763634 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.661788:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763609.661790:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763609.661792:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763609.661795:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763609.661799:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763609.661801:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315996692
00000100:00000001:1.0:1787763609.661803:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315997939
00000100:00000001:1.0:1787763609.661805:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315996692
00000100:00000001:1.0:1787763609.661808:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315997939
00000100:00000001:1.0:1787763609.661810:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763609.661812:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763609.661814:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763609.661818:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763609/real 1787763609]  req@ffff9751f876cb40 x1874599832642176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763609.661827:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763609.661828:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763609.661830:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763609.661832:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763609.661834:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763609.661836:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763609.661840:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cb40 x1874599832642176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763609.661849:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cb40 x1874599832642176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.661856:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763609.661858:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763609.661860:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763609.661861:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763609.661865:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cb40 x1874599832642176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.661872:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763609.661876:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cb40 x1874599832642176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.661885:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763609.661887:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763609.661889:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763609.661890:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763609.661892:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763609.661894:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763609.661897:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763609.661900:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763609.661902:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763609.661907:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cb40 x1874599832642176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763609.661916:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832642176:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763609.661935:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763609.661939:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832642304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.661947:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763609.661949:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763609.661951:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763609.661953:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763609.661955:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763609.661959:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832642304:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763609.661962:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763609.661964:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763609.661966:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763609.661968:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763609.661971:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763609.661974:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763609.661977:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763609.661979:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763609.661982:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763609.661985:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832642304, portal 4
00000100:00000001:1.0:1787763609.661988:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763609.661989:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
00000100:00000040:1.0:1787763609.661995:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832642304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763634 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.662001:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763609.662004:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763609.662006:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763609.662008:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832642304, offset 0
00000400:00000200:1.0:1787763609.662013:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763609.662016:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763609.662019:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763609.662022:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763609.662025:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763609.662028:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763609.662030:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763609.662034:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58253:58253:256:4294967295] 192.168.203.110@tcp LPNI seq info [58253:58253:8:4294967295]
00000400:00000200:1.0:1787763609.662038:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763609.662045:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763609.662050:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763609.662053:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763609.662057:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763609.662060:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787763609.662065:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763609.662067:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787763609.662071:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763609.662073:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763609.662077:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763609.662080:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763609.662083:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763609.662085:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763609.662087:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763609.662090:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876c000 x1874599832642304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763634 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.662097:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763609.662101:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832642304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763634 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.662108:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763609.662110:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763609.662113:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763609.662116:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763609.662119:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763609.662121:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4315996709
00000100:00000001:1.0:1787763609.662124:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4315997939
00000100:00000001:1.0:1787763609.662126:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4315996709
00000100:00000001:1.0:1787763609.662128:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4315997939
00000100:00000001:1.0:1787763609.662131:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763609.662148:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763609.662151:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763609.662156:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763609/real 1787763609]  req@ffff9751f876c000 x1874599832642304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763609.662165:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763609.662166:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763609.662168:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763609.662170:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763609.662172:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763609.662174:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763609.662178:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832642304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763609.662188:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832642304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.662195:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763609.662196:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763609.662198:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763609.662200:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763609.662204:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832642304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.662211:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763609.662215:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832642304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.662223:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763609.662225:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763609.662226:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763609.662228:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763609.662230:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763609.662232:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763609.662235:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763609.662238:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763609.662240:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763609.662245:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832642304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763609.662255:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832642304:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763609.662258:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763609.662260:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763609.662264:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876de00 x1874599832642048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.662272:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763609.662274:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763609.662276:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763609.662278:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763609.662280:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763609.662281:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763609.662284:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763609.662286:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:1.0:1787763609.662288:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763609.662290:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763609.662292:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876de00.
00000100:00000001:1.0:1787763609.662295:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763609.662296:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763609.662298:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763609.662302:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cb40 x1874599832642176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.662309:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763609.662311:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763609.662312:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763609.662314:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763609.662315:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763609.662317:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763609.662319:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763609.662320:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:1.0:1787763609.662322:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763609.662344:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763609.662346:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cb40.
00000100:00000001:1.0:1787763609.662349:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763609.662351:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763609.662352:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763609.662356:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832642304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763634 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763609.662363:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763609.662364:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763609.662366:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763609.662368:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763609.662370:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763609.662371:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763609.662373:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763609.662375:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017000.
02000000:00000001:1.0:1787763609.662377:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763609.662378:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763609.662379:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:1.0:1787763609.662381:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763609.662383:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763609.662385:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763609.662388:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763609.662389:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763609.662391:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763609.662393:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763610.300322:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763610.300326:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763610.300328:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763611.324306:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763611.324310:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763611.324313:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763612.348331:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763612.348337:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763612.348341:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763613.372341:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763613.372347:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763613.372351:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763614.396317:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763614.396322:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763614.396325:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763614.780568:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763614.780576:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763614.780582:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763614.780596:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832642432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.780608:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763614.780611:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763614.780616:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763614.780619:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763614.780624:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763614.780631:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832642432:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763614.780637:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763614.780642:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763614.780647:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763614.780651:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763614.780658:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763614.780664:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763614.780670:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763614.780675:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763614.780684:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763614.780692:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832642432, portal 25
00000100:00000001:1.0:1787763614.780698:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763614.780702:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:1.0:1787763614.780711:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832642432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763639 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.780727:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763614.780733:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763614.780737:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763614.780742:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832642432, offset 0
00000400:00000200:1.0:1787763614.780751:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763614.780762:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763614.780772:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763614.780779:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763614.780786:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763614.780791:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763614.780795:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763614.780800:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58254:58254:256:4294967295] 192.168.203.110@tcp LPNI seq info [58254:58254:8:4294967295]
00000400:00000200:1.0:1787763614.780809:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763614.780825:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763614.780836:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763614.780843:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763614.780855:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763614.780862:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787763614.780871:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763614.780875:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787763614.780882:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763614.780899:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763614.780908:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763614.780932:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763614.780938:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763614.780942:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763614.780949:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763614.780956:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832642432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763639 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.780982:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763614.780993:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832642432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763639 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781014:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763614.781021:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763614.781031:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763614.781040:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763614.781045:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316003059
00000100:00000010:1.0:1787763614.781072:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4ae0.
00000100:00000001:1.0:1787763614.781080:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787763614.781090:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316003059
00000100:00000010:1.0:1787763614.781097:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4840.
00000100:00000001:1.0:1787763614.781101:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763614.781106:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763614.781111:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763614.781114:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763614.781138:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763614/real 1787763614]  req@ffff9751f86583c0 x1874599832642432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763614.781156:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763614.781160:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763614.781165:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763614.781170:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763614.781175:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763614.781199:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763614.781203:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832642432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763614.781211:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832642432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781236:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763614.781239:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763614.781242:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763614.781245:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763614.781254:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832642432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781265:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763614.781271:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832642432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781276:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763614.781279:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763614.781280:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763614.781281:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763614.781283:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763614.781284:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763614.781286:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763614.781289:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763614.781290:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763614.781293:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832642432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763614.781298:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832642432:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763614.781304:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763614.781305:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763614.781308:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832642432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781312:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763614.781314:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763614.781316:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763614.781318:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763614.781320:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763614.781321:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763614.781322:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763614.781324:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:1.0:1787763614.781327:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763614.781328:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763614.781329:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:1.0:1787763614.781331:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763614.781332:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763614.781334:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763614.781339:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763614.781341:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763614.781342:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763614.781343:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763614.781345:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763614.781347:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832642560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781368:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763614.781369:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763614.781371:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763614.781372:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763614.781374:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763614.781376:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832642560:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763614.781377:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763614.781379:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763614.781380:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763614.781382:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763614.781384:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763614.781386:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763614.781389:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763614.781397:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763614.781400:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763614.781402:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832642560, portal 10
00000100:00000001:1.0:1787763614.781404:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763614.781405:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000040:1.0:1787763614.781410:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832642560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763639 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781414:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763614.781416:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763614.781417:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763614.781419:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832642560, offset 0
00000400:00000200:1.0:1787763614.781422:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763614.781425:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763614.781427:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763614.781429:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763614.781431:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763614.781433:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763614.781435:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763614.781437:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58255:58255:256:4294967295] 192.168.203.110@tcp LPNI seq info [58255:58255:8:4294967295]
00000400:00000200:1.0:1787763614.781440:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763614.781444:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763614.781447:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763614.781449:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763614.781452:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763614.781454:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787763614.781457:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763614.781459:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787763614.781462:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763614.781463:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763614.781466:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763614.781468:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763614.781470:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763614.781471:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763614.781474:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763614.781477:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832642560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763639 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781486:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763614.781489:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832642560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763639 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781495:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763614.781497:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763614.781501:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763614.781505:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763614.781509:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763614.781511:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316001824
00000100:00000001:1.0:1787763614.781514:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316003059
00000100:00000001:1.0:1787763614.781516:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316001824
00000100:00000001:1.0:1787763614.781519:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316003059
00000100:00000001:1.0:1787763614.781521:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763614.781523:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763614.781525:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763614.781529:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763614/real 1787763614]  req@ffff9751f8658780 x1874599832642560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763614.781535:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763614.781536:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763614.781537:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763614.781539:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763614.781540:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763614.781542:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763614.781544:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832642560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763614.781553:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832642560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781559:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763614.781561:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763614.781563:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763614.781565:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763614.781569:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832642560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781577:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763614.781581:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832642560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781589:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763614.781592:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763614.781593:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763614.781594:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763614.781596:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763614.781606:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763614.781607:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763614.781610:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763614.781612:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763614.781614:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832642560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763614.781620:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832642560:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763614.781622:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763614.781624:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832642688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781633:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763614.781634:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763614.781636:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763614.781637:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763614.781638:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763614.781640:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832642688:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763614.781642:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763614.781643:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763614.781645:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763614.781646:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763614.781649:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763614.781650:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763614.781653:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763614.781654:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763614.781657:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763614.781659:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832642688, portal 10
00000100:00000001:1.0:1787763614.781660:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763614.781661:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787763614.781665:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832642688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763639 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781669:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763614.781671:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763614.781672:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763614.781674:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832642688, offset 0
00000400:00000200:1.0:1787763614.781677:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763614.781679:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763614.781681:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763614.781684:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763614.781685:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763614.781687:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763614.781689:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763614.781691:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58256:58256:256:4294967295] 192.168.203.110@tcp LPNI seq info [58256:58256:8:4294967295]
00000400:00000200:1.0:1787763614.781694:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763614.781698:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763614.781701:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763614.781703:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763614.781706:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763614.781708:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787763614.781711:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763614.781713:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787763614.781715:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763614.781717:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763614.781719:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763614.781721:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763614.781723:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763614.781725:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763614.781726:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763614.781728:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832642688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763639 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781734:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763614.781736:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832642688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763639 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781740:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763614.781741:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763614.781743:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763614.781745:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763614.781747:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763614.781748:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316001812
00000100:00000001:1.0:1787763614.781749:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316003059
00000100:00000001:1.0:1787763614.781751:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316001812
00000100:00000001:1.0:1787763614.781752:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316003059
00000100:00000001:1.0:1787763614.781754:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763614.781755:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763614.781757:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763614.781759:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763614/real 1787763614]  req@ffff9751f8659a40 x1874599832642688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763614.781764:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763614.781765:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763614.781766:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763614.781767:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763614.781768:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763614.781769:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763614.781771:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832642688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763614.781777:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832642688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781780:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763614.781781:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763614.781782:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763614.781783:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763614.781785:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832642688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781789:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763614.781791:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832642688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781796:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763614.781797:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763614.781798:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763614.781799:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763614.781800:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763614.781801:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763614.781802:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763614.781805:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763614.781806:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763614.781809:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832642688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763614.781814:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832642688:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763614.781816:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763614.781818:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832642816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781822:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763614.781823:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763614.781824:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763614.781825:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763614.781826:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763614.781828:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832642816:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763614.781829:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763614.781830:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763614.781831:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763614.781832:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763614.781835:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763614.781836:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763614.781838:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763614.781839:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763614.781841:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763614.781843:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832642816, portal 4
00000100:00000001:1.0:1787763614.781844:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763614.781845:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787763614.781848:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832642816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763639 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781853:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763614.781855:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763614.781856:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763614.781857:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832642816, offset 0
00000400:00000200:1.0:1787763614.781860:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763614.781862:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763614.781864:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763614.781866:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763614.781868:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763614.781869:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763614.781871:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763614.781873:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58257:58257:256:4294967295] 192.168.203.110@tcp LPNI seq info [58257:58257:8:4294967295]
00000400:00000200:1.0:1787763614.781875:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763614.781879:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763614.781882:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763614.781900:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763614.781903:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763614.781905:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787763614.781908:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763614.781910:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787763614.781912:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763614.781913:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763614.781915:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763614.781933:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763614.781935:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763614.781937:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763614.781939:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763614.781941:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832642816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763639 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781947:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763614.781949:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832642816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763639 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.781953:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763614.781954:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763614.781957:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763614.781959:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763614.781961:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763614.781963:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316001829
00000100:00000001:1.0:1787763614.781965:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316003059
00000100:00000001:1.0:1787763614.781966:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316001829
00000100:00000001:1.0:1787763614.781968:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316003059
00000100:00000001:1.0:1787763614.781970:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763614.781972:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763614.781974:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763614.781978:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763614/real 1787763614]  req@ffff9751f8658000 x1874599832642816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763614.781986:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763614.781988:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763614.781990:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763614.781992:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763614.781994:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763614.781997:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763614.782000:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832642816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763614.782010:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832642816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.782017:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763614.782019:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763614.782021:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763614.782023:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763614.782026:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832642816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.782031:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763614.782034:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832642816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.782039:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763614.782041:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763614.782042:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763614.782043:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763614.782044:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763614.782045:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763614.782047:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763614.782049:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763614.782066:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763614.782068:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832642816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763614.782073:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832642816:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763614.782075:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763614.782077:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763614.782079:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832642560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.782083:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763614.782084:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763614.782085:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763614.782087:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763614.782088:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763614.782089:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763614.782091:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763614.782092:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:1.0:1787763614.782094:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763614.782095:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763614.782096:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:1.0:1787763614.782097:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763614.782098:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763614.782100:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763614.782102:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832642688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.782106:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763614.782107:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763614.782108:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763614.782109:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763614.782109:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763614.782110:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763614.782111:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763614.782112:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:1.0:1787763614.782113:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763614.782114:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763614.782115:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787763614.782116:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763614.782117:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763614.782118:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763614.782120:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832642816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763639 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763614.782123:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763614.782124:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763614.782125:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763614.782126:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763614.782126:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763614.782127:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763614.782128:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763614.782129:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:1.0:1787763614.782130:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763614.782131:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763614.782132:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:1.0:1787763614.782133:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763614.782133:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763614.782135:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763614.782137:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763614.782138:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763614.782139:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763614.782141:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763615.420400:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763615.420405:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763615.420408:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763616.444343:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763616.444347:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763616.444351:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763617.468408:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763617.468412:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763617.468443:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763618.492386:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763618.492392:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763618.492395:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763619.516308:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763619.516311:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763619.516314:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763619.900445:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763619.900450:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763619.900454:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763619.900464:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832642944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.900490:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763619.900492:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763619.900498:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763619.900503:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763619.900506:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763619.900511:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832642944:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763619.900517:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763619.900522:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763619.900525:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763619.900527:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763619.900533:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763619.900537:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763619.900544:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763619.900558:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763619.900567:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763619.900576:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832642944, portal 25
00000100:00000001:1.0:1787763619.900581:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763619.900584:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:1.0:1787763619.900593:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832642944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763644 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.900607:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763619.900614:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763619.900618:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763619.900624:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832642944, offset 0
00000400:00000200:1.0:1787763619.900634:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763619.900647:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763619.900657:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763619.900664:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763619.900671:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763619.900676:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763619.900683:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763619.900689:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58258:58258:256:4294967295] 192.168.203.110@tcp LPNI seq info [58258:58258:8:4294967295]
00000400:00000200:1.0:1787763619.900700:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763619.900714:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763619.900724:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763619.900731:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763619.900746:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763619.900752:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787763619.900763:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763619.900770:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1787763619.900778:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763619.900783:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763619.900789:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763619.900801:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763619.900810:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763619.900816:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763619.900824:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763619.900834:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832642944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763644 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.900860:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763619.900868:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832642944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763644 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.900880:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763619.900883:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763619.900892:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763619.900900:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763619.900904:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316008179
00000100:00000010:1.0:1787763619.900911:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4c00.
00000100:00000001:1.0:1787763619.900921:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787763619.900929:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316008179
00000100:00000010:1.0:1787763619.900936:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c43c0.
00000100:00000001:1.0:1787763619.900942:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763619.900947:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763619.900952:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763619.900959:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763619.900968:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763619/real 1787763619]  req@ffff9751f865ad00 x1874599832642944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763619.900988:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763619.900991:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763619.900995:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763619.901023:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763619.901028:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763619.901033:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763619.901043:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832642944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763619.901105:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832642944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901114:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763619.901116:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763619.901118:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763619.901119:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763619.901122:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832642944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901127:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763619.901131:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832642944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901136:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763619.901137:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763619.901138:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763619.901139:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763619.901158:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763619.901160:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763619.901163:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763619.901167:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763619.901169:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763619.901198:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832642944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763619.901207:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832642944:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763619.901211:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763619.901214:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763619.901216:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832642944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901222:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763619.901225:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763619.901228:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763619.901231:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763619.901233:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763619.901235:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763619.901237:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763619.901240:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:1.0:1787763619.901243:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763619.901245:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763619.901247:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:1.0:1787763619.901250:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763619.901251:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763619.901253:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763619.901260:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763619.901261:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763619.901263:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763619.901265:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763619.901266:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763619.901270:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832643072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901284:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763619.901285:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763619.901288:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763619.901290:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763619.901292:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763619.901295:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832643072:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763619.901298:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763619.901300:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763619.901302:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763619.901304:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763619.901307:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763619.901309:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763619.901314:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763619.901316:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763619.901320:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763619.901323:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832643072, portal 10
00000100:00000001:1.0:1787763619.901325:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763619.901326:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787763619.901333:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832643072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763644 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901339:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763619.901342:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763619.901344:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763619.901346:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832643072, offset 0
00000400:00000200:1.0:1787763619.901350:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763619.901354:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763619.901358:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763619.901361:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763619.901364:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763619.901367:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763619.901369:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763619.901373:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58259:58259:256:4294967295] 192.168.203.110@tcp LPNI seq info [58259:58259:8:4294967295]
00000400:00000200:1.0:1787763619.901377:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763619.901384:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763619.901388:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763619.901391:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763619.901395:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763619.901398:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787763619.901403:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763619.901406:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1787763619.901409:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763619.901411:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763619.901415:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763619.901418:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763619.901421:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763619.901423:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763619.901425:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763619.901429:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832643072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763644 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901437:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763619.901440:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832643072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763644 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901446:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763619.901448:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763619.901450:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763619.901454:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763619.901457:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763619.901459:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316006944
00000100:00000001:1.0:1787763619.901477:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316008179
00000100:00000001:1.0:1787763619.901479:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316006944
00000100:00000001:1.0:1787763619.901481:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316008179
00000100:00000001:1.0:1787763619.901482:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763619.901484:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763619.901485:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763619.901488:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763619/real 1787763619]  req@ffff9751f8658000 x1874599832643072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763619.901493:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763619.901494:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763619.901495:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763619.901497:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763619.901498:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763619.901499:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763619.901501:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832643072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763619.901506:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832643072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901510:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763619.901511:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763619.901512:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763619.901513:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763619.901515:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832643072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901519:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763619.901521:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832643072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901525:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763619.901527:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763619.901528:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763619.901528:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763619.901530:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763619.901538:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763619.901540:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763619.901542:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763619.901544:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763619.901546:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832643072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763619.901551:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832643072:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763619.901553:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763619.901555:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832643200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901559:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763619.901560:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763619.901561:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763619.901562:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763619.901564:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763619.901565:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832643200:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763619.901568:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763619.901569:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763619.901570:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763619.901571:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763619.901573:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763619.901575:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763619.901576:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763619.901578:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763619.901579:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763619.901581:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832643200, portal 10
00000100:00000001:1.0:1787763619.901583:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763619.901584:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787763619.901588:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832643200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763644 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901592:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763619.901594:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763619.901595:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763619.901596:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832643200, offset 0
00000400:00000200:1.0:1787763619.901599:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763619.901601:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763619.901604:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763619.901606:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763619.901607:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763619.901609:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763619.901610:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763619.901612:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58260:58260:256:4294967295] 192.168.203.110@tcp LPNI seq info [58260:58260:8:4294967295]
00000400:00000200:1.0:1787763619.901615:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763619.901619:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763619.901622:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763619.901623:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763619.901626:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763619.901628:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787763619.901630:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763619.901632:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1787763619.901634:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763619.901636:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763619.901637:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763619.901640:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763619.901641:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763619.901643:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763619.901644:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763619.901646:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832643200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763644 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901651:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763619.901653:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832643200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763644 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901657:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763619.901658:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763619.901660:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763619.901662:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763619.901664:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763619.901666:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316006932
00000100:00000001:1.0:1787763619.901667:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316008179
00000100:00000001:1.0:1787763619.901669:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316006932
00000100:00000001:1.0:1787763619.901670:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316008179
00000100:00000001:1.0:1787763619.901671:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763619.901673:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763619.901674:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763619.901676:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763619/real 1787763619]  req@ffff9751f8659a40 x1874599832643200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763619.901680:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763619.901681:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763619.901682:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763619.901683:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763619.901684:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763619.901685:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763619.901687:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832643200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763619.901693:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832643200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901696:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763619.901697:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763619.901698:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763619.901699:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763619.901701:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832643200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901705:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763619.901707:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832643200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901712:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763619.901714:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763619.901715:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763619.901716:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763619.901717:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763619.901718:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763619.901719:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763619.901721:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763619.901722:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763619.901725:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832643200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763619.901729:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832643200:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763619.901731:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763619.901733:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832643328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901737:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763619.901738:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763619.901739:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763619.901740:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763619.901742:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763619.901743:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832643328:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763619.901745:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763619.901746:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763619.901747:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763619.901748:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763619.901751:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763619.901752:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763619.901754:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763619.901755:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763619.901758:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763619.901759:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832643328, portal 4
00000100:00000001:1.0:1787763619.901761:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763619.901762:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000040:1.0:1787763619.901765:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832643328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763644 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901769:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763619.901770:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763619.901772:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763619.901773:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832643328, offset 0
00000400:00000200:1.0:1787763619.901775:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763619.901777:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763619.901779:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763619.901781:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763619.901782:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763619.901784:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763619.901785:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763619.901787:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58261:58261:256:4294967295] 192.168.203.110@tcp LPNI seq info [58261:58261:8:4294967295]
00000400:00000200:1.0:1787763619.901790:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763619.901796:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763619.901801:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763619.901804:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763619.901807:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763619.901810:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787763619.901814:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763619.901817:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1787763619.901821:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763619.901823:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763619.901826:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763619.901829:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763619.901832:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763619.901835:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763619.901838:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763619.901842:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832643328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763644 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901861:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763619.901863:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832643328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763644 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901868:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763619.901869:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763619.901872:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763619.901874:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763619.901876:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763619.901878:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316006949
00000100:00000001:1.0:1787763619.901880:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316008179
00000100:00000001:1.0:1787763619.901881:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316006949
00000100:00000001:1.0:1787763619.901883:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316008179
00000100:00000001:1.0:1787763619.901884:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763619.901885:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763619.901887:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763619.901889:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763619/real 1787763619]  req@ffff9751f8658780 x1874599832643328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763619.901894:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763619.901894:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763619.901896:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e92104d0.
00000400:00000200:1.0:1787763619.901897:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933aa0
00000400:00000010:1.0:1787763619.901899:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933aa0.
00000100:00000001:1.0:1787763619.901900:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763619.901902:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832643328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763619.901907:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832643328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901912:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763619.901913:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763619.901914:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763619.901915:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763619.901917:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832643328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901921:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763619.901923:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832643328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901927:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763619.901929:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763619.901930:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763619.901930:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763619.901931:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763619.901933:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763619.901935:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763619.901938:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763619.901940:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763619.901943:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832643328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763619.901951:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832643328:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763619.901954:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763619.901956:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763619.901959:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832643072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901967:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763619.901968:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763619.901971:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763619.901974:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763619.901976:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763619.901977:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763619.901980:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763619.901981:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:1.0:1787763619.901984:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763619.901985:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763619.901986:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:1.0:1787763619.901988:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763619.901989:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763619.901990:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763619.901993:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832643200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.901997:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763619.901998:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763619.901999:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763619.902000:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763619.902001:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763619.902002:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763619.902003:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763619.902004:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:1.0:1787763619.902006:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763619.902006:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763619.902007:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787763619.902009:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763619.902010:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763619.902011:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763619.902013:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832643328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763644 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763619.902016:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763619.902017:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763619.902018:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763619.902019:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763619.902020:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763619.902020:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763619.902022:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763619.902023:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:1.0:1787763619.902025:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763619.902025:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763619.902026:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:1.0:1787763619.902028:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763619.902028:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763619.902029:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763619.902032:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763619.902033:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763619.902034:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763619.902035:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763620.540412:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763620.540420:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763620.540427:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763621.564447:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763621.564453:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763621.564457:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763622.588391:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763622.588398:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763622.588402:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1787763623.612426:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763623.612431:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763623.612436:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1787763624.636346:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763624.636350:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763624.636354:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763625.020600:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763625.020608:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763625.020612:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763625.020629:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832643456/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763625.020654:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763625.020655:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763625.020663:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763625.020671:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763625.020677:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763625.020691:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832643456:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763625.020699:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763625.020703:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763625.020708:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763625.020714:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763625.020727:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763625.020732:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763625.020747:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763625.020753:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92104d0.
00000400:00000010:1.0:1787763625.020760:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933aa0.
00000100:00000200:1.0:1787763625.020766:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832643456, portal 25
00000100:00000001:1.0:1787763625.020767:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763625.020769:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:1.0:1787763625.020776:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832643456/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763650 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763625.020791:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763625.020794:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763625.020797:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00000200:1.0:1787763625.020807:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832643456, offset 0
00000400:00000200:1.0:1787763625.020832:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763625.020843:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763625.020858:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763625.020869:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763625.020877:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763625.020881:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763625.020888:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763625.020893:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58262:58262:256:4294967295] 192.168.203.110@tcp LPNI seq info [58262:58262:8:4294967295]
00000400:00000200:1.0:1787763625.020903:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763625.020915:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763625.020926:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763625.020936:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763625.020947:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763625.020957:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787763625.020983:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763625.020989:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763625.020996:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316013299
00000100:00000010:1.0:1787763625.021003:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c41e0.
00000100:00000001:1.0:1787763625.021012:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787763625.021024:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316013299
00000100:00000010:1.0:1787763625.021032:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4480.
00000100:00000001:1.0:1787763625.021040:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763625.021048:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763625.021053:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763625.021056:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763625.021060:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763625.021067:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763625.021072:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1787763625.021076:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763625.021081:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763625.021083:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763625.021096:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832643584/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763625.021114:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763625.021118:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763625.021122:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763625.021126:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763625.021131:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763625.021191:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832643584:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763625.021197:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763625.021201:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763625.021203:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763625.021209:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763625.021215:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763625.021219:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763625.021228:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763625.021239:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e92101b8.
00000400:00000010:1.0:1787763625.021246:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933990.
00000100:00000200:1.0:1787763625.021251:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832643584, portal 10
00000100:00000001:1.0:1787763625.021255:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763625.021258:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000040:1.0:1787763625.021272:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832643584/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763650 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763625.021282:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763625.021291:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763625.021294:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933d48.
00000100:00000200:1.0:1787763625.021300:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832643584, offset 0
00000400:00000200:1.0:1787763625.021307:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763625.021317:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763625.021326:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763625.021336:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763625.021344:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763625.021352:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763625.021363:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763625.021373:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58263:58263:255:4294967295] 192.168.203.110@tcp LPNI seq info [58263:58263:7:4294967295]
00000400:00000200:1.0:1787763625.021384:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763625.021394:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763625.021407:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763625.021413:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1300.
00000800:00000200:1.0:1787763625.021425:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763625.021437:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787763625.021446:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763625.021453:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763625.021460:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763625.021465:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316012064
00000100:00000001:1.0:1787763625.021477:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316013299
00000100:00000001:1.0:1787763625.021481:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316012064
00000100:00000001:1.0:1787763625.021485:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316013299
00000100:00000001:1.0:1787763625.021490:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763625.021498:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763625.021505:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763625.021514:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763625.021523:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763625.021528:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1787763625.021533:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763625.021539:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763625.021543:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763625.021558:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832643712/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763625.021598:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763625.021607:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763625.021614:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763625.021620:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763625.021630:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763625.021649:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832643712:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763625.021675:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763625.021684:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763625.021689:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763625.021702:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763625.021714:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763625.021724:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763625.021742:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763625.021750:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210840.
00000400:00000010:1.0:1787763625.021764:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933b28.
00000100:00000200:1.0:1787763625.021776:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832643712, portal 10
00000100:00000001:1.0:1787763625.021784:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763625.021790:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787763625.021828:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832643712/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763650 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763625.021857:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763625.021875:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763625.021884:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763625.021897:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832643712, offset 0
00000400:00000200:1.0:1787763625.021918:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763625.021929:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763625.021946:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763625.021963:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763625.021980:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763625.021989:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763625.022005:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763625.022029:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58264:58264:254:4294967295] 192.168.203.110@tcp LPNI seq info [58264:58264:6:4294967295]
00000400:00000200:1.0:1787763625.022045:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763625.022068:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763625.022077:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763625.022085:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1b00.
00000800:00000200:1.0:1787763625.022095:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763625.022101:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787763625.022105:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763625.022109:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763625.022113:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763625.022115:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316012052
00000100:00000001:1.0:1787763625.022118:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316013299
00000100:00000001:1.0:1787763625.022120:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316012052
00000100:00000001:1.0:1787763625.022122:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316013299
00000100:00000001:1.0:1787763625.022125:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763625.022127:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763625.022130:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763625.022176:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763625.022185:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763625.022187:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1787763625.022191:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763625.022194:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763625.022196:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763625.022204:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832643840/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763625.022212:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763625.022214:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763625.022218:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763625.022221:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763625.022224:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763625.022228:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832643840:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763625.022233:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763625.022236:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763625.022238:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763625.022240:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763625.022247:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787763625.022251:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763625.022256:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763625.022261:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210e70.
00000400:00000010:1.0:1787763625.022268:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933440.
00000100:00000200:1.0:1787763625.022273:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832643840, portal 4
00000100:00000001:1.0:1787763625.022277:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763625.022279:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787763625.022287:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832643840/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763650 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763625.022294:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763625.022298:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763625.022302:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763625.022305:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832643840, offset 0
00000400:00000200:1.0:1787763625.022311:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763625.022315:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763625.022319:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763625.022322:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763625.022324:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763625.022326:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763625.022328:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763625.022332:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58265:58265:253:4294967295] 192.168.203.110@tcp LPNI seq info [58265:58265:5:4294967295]
00000400:00000200:1.0:1787763625.022339:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763625.022349:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763625.022355:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763625.022361:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763625.022365:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763625.022372:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787763625.022376:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763625.022379:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763625.022382:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763625.022385:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316012070
00000100:00000001:1.0:1787763625.022388:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316013299
00000100:00000001:1.0:1787763625.022391:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316012070
00000100:00000001:1.0:1787763625.022394:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316013299
00000100:00000001:1.0:1787763625.022396:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763625.022398:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763625.022401:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763625.022403:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763625.022407:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763625.022408:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1787763625.022410:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763625.022412:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763625.022414:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763625.022417:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763625.660418:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763625.660424:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763625.660427:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1787763626.684403:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763626.684410:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763626.684415:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1787763627.708343:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763627.708347:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763627.708350:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000400:00000100:1.0:1787763628.092477:0:257564:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.110:988
00000400:00000100:1.0:1787763628.092529:0:257564:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.110@tcp at host 192.168.203.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:1.0:1787763628.092540:0:257564:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763628.092543:0:257564:0:(router.c:1791:lnet_notify()) 192.168.203.10@tcp notifying 192.168.203.110@tcp: down
00000400:00000200:1.0:1787763628.092546:0:257564:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763628.092551:0:257564:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:1.0:1787763628.092555:0:257564:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:1.0:1787763628.092559:0:257564:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000400:00000200:1.0:1787763628.092564:0:257564:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1787763628.092569:0:257564:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1787763628.092577:0:257564:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:1.0:1787763628.092581:0:257564:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763628.092587:0:257564:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763628.092598:0:257564:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f86583c0 x1874599832643456/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763650 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763628.092627:0:257564:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763628.092632:0:257564:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832643456/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763650 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763628.092641:0:257564:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763628.092643:0:257564:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1787763628.092657:0:257564:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:1.0:1787763628.092660:0:257564:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1300.
00000400:00000200:1.0:1787763628.092664:0:257564:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1787763628.092670:0:257564:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1787763628.092675:0:257564:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933d48
00000400:00000010:1.0:1787763628.092680:0:257564:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933d48.
00000100:00000001:1.0:1787763628.092685:0:257564:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763628.092697:0:257564:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f8658780 x1874599832643584/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763650 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763628.092732:0:257564:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763628.092738:0:257564:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832643584/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763650 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763628.092748:0:257564:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763628.092752:0:257564:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1787763628.092763:0:257564:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:1.0:1787763628.092768:0:257564:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1b00.
00000400:00000200:1.0:1787763628.092773:0:257564:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1787763628.092781:0:257564:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1787763628.092787:0:257564:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763628.092807:0:257564:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763628.092815:0:257564:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763628.092823:0:257564:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f8659a40 x1874599832643712/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763650 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763628.092850:0:257564:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763628.092856:0:257564:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832643712/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763650 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763628.092866:0:257564:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763628.092870:0:257564:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:1.0:1787763628.092882:0:257564:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:1.0:1787763628.092888:0:257564:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763628.092896:0:257564:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:1.0:1787763628.092905:0:257564:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:1.0:1787763628.092911:0:257564:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763628.092916:0:257564:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763628.092922:0:257564:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763628.092930:0:257564:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f8658000 x1874599832643840/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763650 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763628.092958:0:257564:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763628.092963:0:257564:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832643840/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763650 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763628.092976:0:257564:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763628.092984:0:257564:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:0.0:1787763628.732348:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763628.732352:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763628.732355:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1787763629.756338:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763629.756342:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763629.756345:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787763630.140849:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763630.140857:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787763630.141504:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763630.141507:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787763630.141835:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763630.141838:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787763630.142101:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763630.142103:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:1.0:1787763630.780324:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763630.780328:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763630.780331:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763631.804387:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763631.804393:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763631.804399:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763632.188306:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763632.188318:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763632.188324:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763632.188337:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763632.188345:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763632.188351:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763632.188357:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763632.188362:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763632.188367:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763632.828405:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763632.828412:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763632.828417:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763633.212306:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763633.212320:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763633.212325:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763633.212334:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763633.212342:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763633.212350:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763633.212355:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763633.212360:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763633.212365:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763633.852336:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763633.852341:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763633.852344:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763634.236299:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763634.236307:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763634.236310:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763634.236317:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763634.236322:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763634.236327:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763634.236330:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763634.236333:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763634.236336:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763635.260292:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763635.260303:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787763635.260307:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763635.260312:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763635.260318:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763635.260328:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21349
00000400:00000200:1.0:1787763635.260344:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763635.260357:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763635.260364:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763635.260372:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763635.260378:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763635.260382:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763635.260387:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763635.260392:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763635.260396:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787763635.260400:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763635.260404:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787763635.260407:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787763635.260409:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787763635.260412:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763635.260414:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763635.260416:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763635.260418:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763635.260420:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787763635.260423:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763635.260437:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763635.260440:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763635.260444:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763635.260446:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763635.260449:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763635.260452:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763635.260465:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932940.
00000020:00000040:1.0:1787763635.260481:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787763635.260485:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763635.260487:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763635.260491:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763635.260494:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763635.260500:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763635.260510:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8400.
00000100:00000001:1.0:1787763635.260519:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763635.260531:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932940 x1874599832644480/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763635.260544:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763635.260550:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763635.260557:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000af90eaa] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932940 x1874599832644480/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763635.260609:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763635.260614:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763635.260631:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787763635.260633:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763635.260636:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763635.260641:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763635.260645:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21349
00000400:00000200:1.0:1787763635.260653:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763635.260660:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763635.260664:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763635.260669:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763635.260674:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763635.260683:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763635.260692:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763635.260700:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763635.260705:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787763635.260711:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763635.260718:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787763635.260727:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787763635.260737:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787763635.260787:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763635.260793:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763635.260801:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763635.260810:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763635.260819:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787763635.260828:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763635.260836:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763635.260844:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763635.260861:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763635.260868:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763635.260875:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763635.260882:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763635.260889:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763635.260894:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763635.260901:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930780.
00000020:00000040:1.0:1787763635.260906:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763635.260909:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763635.260911:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763635.260914:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763635.260917:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763635.260922:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763635.260928:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8000.
00000100:00000001:1.0:1787763635.260936:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763635.260946:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930780 x1874599832644608/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763635.260966:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763635.260982:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763635.260998:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3977950] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930780 x1874599832644608/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763635.261025:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763635.261042:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763635.261053:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787763635.261058:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763635.261064:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763635.261075:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763635.261085:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21349
00000400:00000200:1.0:1787763635.261103:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763635.261116:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763635.261172:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763635.261183:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763635.261192:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.1:1787763635.261198:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:1.0:1787763635.261203:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000200:3.1:1787763635.261204:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:1.0:1787763635.261214:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763635.261223:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763635.261228:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787763635.261235:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763635.261243:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787763635.261252:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787763635.261259:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787763635.261267:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763635.261274:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763635.261283:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763635.261289:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763635.261295:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787763635.261301:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763635.261306:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763635.261310:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763635.261316:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763635.261323:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763635.261328:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763635.261332:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763635.261339:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763635.261346:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763635.261354:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932580.
00000020:00000040:1.0:1787763635.261363:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763635.261370:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763635.261374:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763635.261379:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763635.261384:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763635.261387:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763635.261396:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb400.
00000100:00000001:1.0:1787763635.261431:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763635.261439:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932580 x1874599832644736/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763635.261450:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763635.261456:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763635.261467:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d026924] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932580 x1874599832644736/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763635.261510:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763635.261514:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787763635.261519:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787763635.261526:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763635.261546:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787763635.261549:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763635.261553:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763635.261559:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763635.261566:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21349
00000400:00000200:1.0:1787763635.261576:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763635.261583:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763635.261587:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763635.261612:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763635.261619:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763635.261623:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763635.261636:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763635.261642:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763635.261644:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787763635.261649:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763635.261653:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787763635.261658:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787763635.261664:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:1.0:1787763635.261672:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763635.261678:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763635.261686:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763635.261688:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763635.261693:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:1.0:1787763635.261701:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763635.261709:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763635.261715:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763635.261721:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763635.261726:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763635.261732:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763635.261740:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1787763635.261771:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763635.261778:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763635.261808:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933840.
00000020:00000040:1.0:1787763635.261815:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787763635.261824:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763635.261829:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763635.261835:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763635.261839:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763635.261844:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763635.261849:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca000.
00000100:00000001:1.0:1787763635.261853:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763635.261862:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933840 x1874599832644864/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763635.261899:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763635.261906:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763635.261914:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000083dd6d32] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933840 x1874599832644864/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763635.261936:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763635.261943:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21359)
00000800:00000200:3.1:1787763635.262187:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763635.262191:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787763635.262538:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763635.262540:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787763635.262756:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763635.262759:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:1.0:1787763635.900439:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763635.900443:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763635.900447:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1787763636.924488:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763636.924494:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763636.924499:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1787763638.972333:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763638.972337:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763638.972341:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763639.996322:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763639.996326:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763639.996330:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787763640.380289:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763640.380300:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787763640.380305:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763640.380312:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763640.380320:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763640.380333:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21354
00000400:00000200:1.0:1787763640.380350:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763640.380365:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763640.380372:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763640.380379:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763640.380385:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763640.380390:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763640.380394:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763640.380401:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763640.380405:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787763640.380409:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763640.380413:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787763640.380417:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787763640.380420:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787763640.380424:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763640.380427:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763640.380430:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763640.380432:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763640.380437:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787763640.380441:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763640.380444:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763640.380448:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763640.380453:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763640.380457:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763640.380460:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763640.380464:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763640.380476:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933840.
00000020:00000040:1.0:1787763640.380480:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787763640.380485:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763640.380487:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763640.380491:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763640.380494:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763640.380500:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763640.380527:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9000.
00000100:00000001:1.0:1787763640.380533:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763640.380541:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933840 x1874599832644992/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763640.380550:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763640.380553:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763640.380557:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000083dd6d32] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933840 x1874599832644992/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763640.380599:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763640.380606:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763640.380613:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787763640.380615:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763640.380619:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763640.380625:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763640.380630:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21354
00000400:00000200:1.0:1787763640.380639:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763640.380645:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763640.380649:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763640.380655:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763640.380659:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763640.380664:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763640.380667:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763640.380672:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763640.380674:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787763640.380677:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763640.380682:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787763640.380686:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787763640.380691:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787763640.380698:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763640.380702:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763640.380706:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763640.380735:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763640.380740:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787763640.380747:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763640.380752:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763640.380756:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763640.380764:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763640.380768:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763640.380772:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763640.380779:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763640.380784:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763640.380788:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763640.380795:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932580.
00000020:00000040:1.0:1787763640.380800:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763640.380805:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763640.380807:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763640.380811:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763640.380814:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763640.380818:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763640.380825:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8800.
00000100:00000001:1.0:1787763640.380847:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763640.380853:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932580 x1874599832645120/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763640.380862:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763640.380865:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763640.380869:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d026924] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932580 x1874599832645120/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763640.380879:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763640.380882:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763640.380885:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787763640.380887:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763640.380888:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763640.380891:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763640.380894:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21354
00000400:00000200:1.0:1787763640.380915:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763640.380921:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763640.380925:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763640.380930:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763640.380934:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763640.380938:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763640.380943:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763640.380947:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763640.380950:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787763640.380953:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763640.380957:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787763640.380962:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787763640.380966:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787763640.380971:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763640.380975:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763640.380980:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763640.380986:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763640.380991:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787763640.380999:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763640.381004:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763640.381007:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763640.381013:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763640.381018:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763640.381022:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763640.381026:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763640.381031:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763640.381032:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787763640.381035:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763640.381039:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000100:00000010:1.0:1787763640.381041:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930780.
00000020:00000040:1.0:1787763640.381047:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763640.381051:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763640.381054:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763640.381057:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763640.381060:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763640.381063:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763640.381069:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb800.
00000100:00000001:1.0:1787763640.381075:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763640.381084:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930780 x1874599832645248/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763640.381100:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763640.381104:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763640.381112:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3977950] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930780 x1874599832645248/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763640.381144:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763640.381147:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787763640.381151:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787763640.381155:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763640.381157:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787763640.381159:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763640.381161:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763640.381163:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763640.381166:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21354
00000400:00000200:1.0:1787763640.381170:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763640.381173:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763640.381175:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763640.381178:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763640.381227:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763640.381234:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763640.381241:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763640.381247:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763640.381250:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787763640.381255:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763640.381259:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787763640.381264:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787763640.381268:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:1.0:1787763640.381274:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763640.381277:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763640.381282:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763640.381286:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763640.381291:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:1.0:1787763640.381297:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763640.381301:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763640.381304:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763640.381311:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763640.381316:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763640.381320:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763640.381325:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1787763640.381330:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763640.381334:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763640.381340:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932940.
00000020:00000040:1.0:1787763640.381345:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787763640.381350:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763640.381353:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763640.381356:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763640.381359:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763640.381363:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763640.381370:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8c00.
00000100:00000001:1.0:1787763640.381376:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763640.381385:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932940 x1874599832645376/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763640.381414:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763640.381418:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763640.381426:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000af90eaa] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932940 x1874599832645376/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763640.381443:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763640.381452:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21365)
00000800:00000200:3.1:1787763640.381796:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763640.381798:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787763640.382195:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763640.382197:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787763640.382629:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763640.382633:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000100:00000001:1.0:1787763642.044366:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763642.044382:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763642.044385:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763642.044392:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763642.044419:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763642.044434:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763642.044451:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763642.044458:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763642.044464:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763643.068309:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763643.068320:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763643.068326:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763643.068333:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763643.068342:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763643.068348:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763643.068353:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763643.068375:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763643.068379:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763643.068424:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763643.068428:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763643.068431:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763644.092270:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763644.092279:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763644.092284:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763644.092291:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763644.092297:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763644.092305:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763644.092313:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763644.092318:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763644.092322:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763645.116311:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763645.116324:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763645.116335:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763645.116345:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763645.116353:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763645.116359:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763645.116365:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763645.116370:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763645.116376:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763645.500256:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763645.500267:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787763645.500272:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763645.500277:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763645.500282:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763645.500290:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21360
00000400:00000200:1.0:1787763645.500302:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763645.500310:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763645.500316:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763645.500320:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763645.500325:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763645.500328:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763645.500330:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763645.500334:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763645.500335:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787763645.500338:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763645.500341:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787763645.500343:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787763645.500346:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787763645.500349:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763645.500350:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763645.500352:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763645.500353:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763645.500356:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787763645.500358:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763645.500361:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763645.500364:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763645.500366:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763645.500369:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763645.500371:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763645.500373:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763645.500381:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7931680.
00000020:00000040:1.0:1787763645.500384:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787763645.500388:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763645.500389:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763645.500392:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763645.500395:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763645.500398:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763645.500405:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca800.
00000100:00000001:1.0:1787763645.500411:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763645.500429:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7931680 x1874599832645504/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763645.500438:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763645.500442:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763645.500446:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d4255d6] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7931680 x1874599832645504/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763645.500466:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763645.500469:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763645.500472:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787763645.500473:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763645.500474:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763645.500477:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763645.500479:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21360
00000400:00000200:1.0:1787763645.500481:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763645.500484:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763645.500485:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763645.500488:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763645.500491:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763645.500493:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763645.500495:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763645.500497:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763645.500499:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787763645.500500:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763645.500502:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787763645.500504:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787763645.500506:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787763645.500509:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763645.500514:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763645.500516:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763645.500520:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763645.500522:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787763645.500524:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763645.500526:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763645.500527:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763645.500533:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763645.500538:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763645.500540:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763645.500545:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763645.500552:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763645.500573:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763645.500582:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930000.
00000020:00000040:1.0:1787763645.500590:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763645.500599:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763645.500602:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763645.500610:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763645.500615:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763645.500621:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763645.500630:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca400.
00000100:00000001:1.0:1787763645.500637:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763645.500654:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930000 x1874599832645632/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763645.500707:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763645.500711:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763645.500716:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6d90274] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930000 x1874599832645632/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763645.500729:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763645.500735:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763645.500738:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787763645.500742:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763645.500745:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763645.500749:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763645.500751:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21360
00000400:00000200:1.0:1787763645.500758:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763645.500762:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763645.500764:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763645.500769:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763645.500774:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763645.500778:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763645.500784:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763645.500793:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763645.500798:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787763645.500803:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763645.500809:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787763645.500816:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787763645.500823:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787763645.500832:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763645.500839:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763645.500845:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763645.500848:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763645.500852:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787763645.500860:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763645.500868:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763645.500874:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763645.500881:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763645.500890:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763645.500896:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763645.500902:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763645.500909:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763645.500914:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763645.500922:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933480.
00000020:00000040:1.0:1787763645.500927:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763645.500932:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763645.500935:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763645.500938:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763645.500939:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763645.500941:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763645.500950:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8c00.
00000100:00000001:1.0:1787763645.500955:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763645.500964:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933480 x1874599832645760/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1787763645.500970:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787763645.500979:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:3.1:1787763645.500980:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000100:00000001:1.0:1787763645.500982:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763645.500991:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3c2c33c] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933480 x1874599832645760/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763645.501006:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763645.501010:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787763645.501015:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787763645.501021:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763645.501026:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787763645.501029:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763645.501033:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763645.501036:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763645.501042:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21360
00000400:00000200:1.0:1787763645.501050:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763645.501058:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763645.501062:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763645.501066:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763645.501072:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763645.501079:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763645.501085:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763645.501090:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763645.501093:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787763645.501097:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763645.501100:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787763645.501102:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787763645.501106:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:1.0:1787763645.501113:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763645.501116:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763645.501120:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763645.501123:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763645.501127:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:1.0:1787763645.501131:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763645.501154:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763645.501156:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763645.501158:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763645.501160:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763645.501162:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763645.501165:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1787763645.501167:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763645.501169:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763645.501172:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930b40.
00000020:00000040:1.0:1787763645.501178:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787763645.501181:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763645.501183:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763645.501184:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763645.501188:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763645.501191:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763645.501196:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb800.
00000100:00000001:1.0:1787763645.501201:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763645.501207:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930b40 x1874599832645888/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763645.501217:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763645.501220:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763645.501225:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022d7f91a] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930b40 x1874599832645888/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763645.501238:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763645.501244:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21370)
00000800:00000200:3.1:1787763645.501509:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763645.501512:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787763645.501764:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763645.501766:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787763645.502001:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763645.502003:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000100:00000001:1.0:1787763646.140332:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763646.140342:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763646.140347:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763646.140355:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763646.140366:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763646.140390:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763646.140392:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763646.140394:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763646.140398:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763646.140538:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763646.140543:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763646.140549:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1787763648.188355:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763648.188360:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763648.188364:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763649.212334:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763649.212340:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763649.212344:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787763650.620235:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763650.620245:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787763650.620250:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763650.620256:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763650.620262:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763650.620271:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21365
00000400:00000200:1.0:1787763650.620284:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763650.620292:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763650.620296:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763650.620300:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763650.620304:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763650.620307:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763650.620309:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763650.620314:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763650.620316:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787763650.620319:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763650.620323:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787763650.620325:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787763650.620328:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787763650.620331:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763650.620333:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763650.620335:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763650.620338:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763650.620341:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787763650.620344:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763650.620346:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763650.620350:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763650.620353:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763650.620356:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763650.620358:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763650.620361:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763650.620369:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930b40.
00000020:00000040:1.0:1787763650.620373:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787763650.620377:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763650.620379:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763650.620382:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763650.620385:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763650.620389:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763650.620399:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9000.
00000100:00000001:1.0:1787763650.620415:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763650.620425:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930b40 x1874599832646016/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763650.620437:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763650.620441:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763650.620447:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022d7f91a] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930b40 x1874599832646016/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763650.620468:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763650.620472:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763650.620475:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787763650.620476:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763650.620478:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763650.620482:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763650.620485:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21365
00000400:00000200:1.0:1787763650.620490:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763650.620496:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763650.620500:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763650.620507:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763650.620512:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763650.620517:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763650.620519:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763650.620522:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763650.620525:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787763650.620527:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763650.620531:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787763650.620536:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787763650.620543:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787763650.620550:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763650.620557:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763650.620563:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763650.620568:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763650.620572:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787763650.620579:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763650.620584:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763650.620597:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763650.620604:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763650.620609:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763650.620612:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763650.620619:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763650.620625:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763650.620630:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763650.620648:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933480.
00000020:00000040:1.0:1787763650.620653:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763650.620660:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763650.620663:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763650.620668:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763650.620672:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763650.620676:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763650.620684:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca000.
00000100:00000001:1.0:1787763650.620690:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763650.620700:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933480 x1874599832646144/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763650.620717:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763650.620722:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763650.620734:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3c2c33c] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933480 x1874599832646144/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763650.620770:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763650.620775:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763650.620792:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000800:00000200:3.1:1787763650.620796:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787763650.620797:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763650.620801:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:3.1:1787763650.620802:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000100:00000001:1.0:1787763650.620808:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763650.620816:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21365
00000400:00000200:1.0:1787763650.620824:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763650.620832:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763650.620837:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763650.620842:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763650.620846:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763650.620851:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763650.620857:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763650.620861:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763650.620863:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787763650.620866:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763650.620870:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787763650.620875:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787763650.620880:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787763650.620886:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763650.620889:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763650.620895:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763650.620899:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763650.620904:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787763650.620912:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763650.620917:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763650.620922:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763650.620927:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763650.620934:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763650.620938:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763650.620941:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763650.620947:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763650.620951:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763650.620957:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930000.
00000020:00000040:1.0:1787763650.620962:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763650.620967:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763650.620969:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763650.620973:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763650.620975:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763650.620978:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763650.620985:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb400.
00000100:00000001:1.0:1787763650.620992:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763650.621001:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930000 x1874599832646272/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763650.621026:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763650.621031:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763650.621042:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6d90274] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930000 x1874599832646272/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763650.621059:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763650.621065:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787763650.621074:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787763650.621081:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763650.621088:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787763650.621091:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763650.621094:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763650.621101:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763650.621109:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21365
00000400:00000200:1.0:1787763650.621118:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763650.621126:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763650.621153:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763650.621159:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763650.621161:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763650.621164:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763650.621166:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763650.621169:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763650.621173:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787763650.621178:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763650.621182:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787763650.621188:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787763650.621193:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:1.0:1787763650.621203:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763650.621207:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763650.621211:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763650.621217:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763650.621222:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:1.0:1787763650.621229:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763650.621235:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763650.621239:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763650.621243:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763650.621249:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763650.621254:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763650.621260:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1787763650.621266:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763650.621271:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763650.621278:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7931680.
00000020:00000040:1.0:1787763650.621284:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787763650.621291:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763650.621296:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763650.621302:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763650.621307:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763650.621311:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763650.621319:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8000.
00000100:00000001:1.0:1787763650.621327:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763650.621338:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7931680 x1874599832646400/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763650.621357:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763650.621362:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763650.621376:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d4255d6] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7931680 x1874599832646400/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763650.621398:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763650.621405:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21375)
00000800:00000200:3.1:1787763650.621493:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763650.621496:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787763650.621713:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763650.621714:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787763650.621998:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763650.622000:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000020:00000001:1.0:1787763651.260281:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763651.260285:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763651.260289:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763652.284255:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763652.284260:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763652.284264:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1787763654.332300:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763654.332311:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763654.332317:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763655.356463:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763655.356470:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763655.356475:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763655.740279:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763655.740292:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763655.740296:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763655.740300:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763655.740306:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763655.740314:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21370
00000400:00000200:0.0:1787763655.740323:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763655.740330:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763655.740334:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763655.740338:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763655.740342:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763655.740345:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763655.740348:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763655.740352:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763655.740354:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763655.740358:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763655.740362:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763655.740365:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763655.740368:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763655.740372:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763655.740374:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763655.740377:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763655.740380:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763655.740383:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763655.740388:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763655.740391:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763655.740395:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763655.740399:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763655.740403:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763655.740406:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763655.740409:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763655.740419:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763655.740423:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763655.740428:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763655.740430:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763655.740433:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763655.740436:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763655.740440:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763655.740449:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763655.740456:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763655.740466:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832646528/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763655.740479:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763655.740484:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763655.740490:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832646528/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763655.740515:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763655.740518:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763655.740523:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763655.740525:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763655.740527:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763655.740531:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763655.740536:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21370
00000400:00000200:0.0:1787763655.740544:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763655.740552:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763655.740559:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763655.740569:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763655.740576:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763655.740584:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763655.740590:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763655.740595:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763655.740599:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763655.740620:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763655.740625:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763655.740630:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763655.740638:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763655.740648:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763655.740654:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763655.740659:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763655.740663:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763655.740669:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763655.740677:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763655.740683:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763655.740690:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763655.740699:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763655.740707:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763655.740714:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763655.740721:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763655.740726:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763655.740732:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763655.740739:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763655.740746:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763655.740752:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763655.740757:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763655.740763:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763655.740767:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763655.740773:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763655.740780:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787763655.740788:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763655.740802:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832646656/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1787763655.740823:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763655.740827:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:3.1:1787763655.740828:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:0.0:1787763655.740833:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763655.740844:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832646656/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763655.740869:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763655.740875:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763655.740883:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763655.740887:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763655.740892:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763655.740897:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763655.740905:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21370
00000400:00000200:0.0:1787763655.740917:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763655.740926:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763655.740930:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763655.740940:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763655.740946:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763655.740952:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763655.740957:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763655.740962:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763655.740965:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763655.740970:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763655.740975:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763655.740982:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763655.740989:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763655.740997:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763655.741003:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763655.741009:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763655.741015:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763655.741023:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763655.741032:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763655.741039:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763655.741045:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763655.741052:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763655.741061:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763655.741066:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763655.741072:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763655.741080:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763655.741085:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763655.741093:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763655.741100:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763655.741107:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763655.741112:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763655.741118:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763655.741123:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763655.741128:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763655.741179:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763655.741187:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763655.741199:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832646784/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763655.741217:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763655.741222:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763655.741232:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832646784/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763655.741251:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763655.741257:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763655.741268:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763655.741276:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763655.741284:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763655.741290:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763655.741296:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763655.741304:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763655.741312:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21370
00000400:00000200:0.0:1787763655.741323:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763655.741331:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763655.741338:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763655.741347:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763655.741353:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763655.741360:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763655.741368:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763655.741374:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763655.741378:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763655.741382:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763655.741387:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763655.741394:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763655.741402:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763655.741412:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763655.741419:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763655.741427:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763655.741430:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763655.741433:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763655.741438:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763655.741445:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763655.741452:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763655.741459:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763655.741466:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763655.741472:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763655.741477:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763655.741484:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763655.741490:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763655.741498:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763655.741506:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000800:00000200:3.1:1787763655.741513:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:0.0:1787763655.741514:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:3.1:1787763655.741516:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
02000000:00000001:0.0:1787763655.741518:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763655.741525:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763655.741529:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763655.741534:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763655.741544:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763655.741552:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763655.741564:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832646912/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763655.741592:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763655.741598:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763655.741623:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832646912/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763655.741655:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763655.741663:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21380)
00000800:00000200:3.1:1787763655.741833:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763655.741834:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787763655.742078:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763655.742080:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:0.0:1787763656.380374:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763656.380380:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763656.380384:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1787763657.404313:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763657.404318:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763657.404321:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763658.428337:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763658.428342:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763658.428345:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763659.452364:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763659.452370:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763659.452373:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1787763660.476339:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763660.476343:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763660.476346:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763660.860226:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763660.860235:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763660.860241:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763660.860245:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763660.860250:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763660.860260:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21375
00000400:00000200:0.0:1787763660.860275:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763660.860287:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763660.860290:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763660.860295:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763660.860299:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763660.860306:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763660.860309:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763660.860314:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763660.860316:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763660.860318:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763660.860322:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763660.860325:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763660.860327:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763660.860337:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763660.860340:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763660.860343:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763660.860345:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763660.860348:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763660.860350:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763660.860352:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763660.860355:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763660.860358:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763660.860360:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763660.860362:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763660.860364:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763660.860373:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
00000020:00000040:0.0:1787763660.860376:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763660.860380:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763660.860387:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763660.860390:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763660.860393:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763660.860397:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763660.860408:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763660.860416:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763660.860424:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832647040/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763660.860433:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763660.860437:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763660.860441:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832647040/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763660.860459:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763660.860461:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763660.860464:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763660.860465:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763660.860467:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763660.860469:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763660.860470:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21375
00000400:00000200:0.0:1787763660.860473:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763660.860475:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763660.860477:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763660.860479:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763660.860481:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763660.860483:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763660.860485:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763660.860487:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763660.860488:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763660.860490:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763660.860491:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763660.860493:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763660.860497:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763660.860500:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763660.860504:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763660.860512:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763660.860514:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763660.860517:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763660.860522:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763660.860527:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763660.860531:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763660.860539:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763660.860543:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763660.860545:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763660.860551:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763660.860557:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763660.860560:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763660.860578:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787763660.860583:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763660.860589:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763660.860594:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763660.860598:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763660.860601:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763660.860607:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763660.860615:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763660.860622:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763660.860632:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832647168/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763660.860642:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763660.860646:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763660.860656:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832647168/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763660.860686:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763660.860691:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763660.860695:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763660.860699:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763660.860702:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763660.860705:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763660.860709:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21375
00000400:00000200:0.0:1787763660.860715:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763660.860718:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763660.860719:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763660.860723:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763660.860727:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763660.860730:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763660.860732:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763660.860736:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763660.860739:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763660.860741:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763660.860743:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763660.860746:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763660.860750:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763660.860756:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763660.860759:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763660.860766:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763660.860770:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763660.860774:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763660.860781:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763660.860786:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763660.860788:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763660.860791:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763660.860795:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763660.860801:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763660.860804:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763660.860807:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763660.860812:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763660.860817:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000020:00000040:0.0:1787763660.860821:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763660.860826:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763660.860827:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763660.860828:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763660.860830:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763660.860833:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763660.860839:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763660.860844:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763660.860849:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00100000:0.0:1787763660.860852:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832647296/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1787763660.860854:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:0.0:1787763660.860866:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763660.860869:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763660.860875:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832647296/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763660.860884:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763660.860887:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763660.860892:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763660.860896:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763660.860900:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763660.860902:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763660.860906:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763660.860908:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763660.860913:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21375
00000400:00000200:0.0:1787763660.860921:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763660.860927:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763660.860930:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763660.860936:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763660.860941:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763660.860945:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763660.860950:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763660.860955:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763660.860958:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763660.860959:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763660.860960:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763660.860962:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763660.860966:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763660.860973:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763660.860975:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763660.860979:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763660.860981:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763660.860982:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763660.860988:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763660.860989:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763660.860990:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763660.860993:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763660.860997:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763660.860999:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763660.861001:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763660.861004:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763660.861007:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763660.861012:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763660.861015:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763660.861019:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763660.861023:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763660.861026:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763660.861028:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763660.861031:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763660.861036:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787763660.861041:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763660.861050:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832647424/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763660.861061:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763660.861065:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763660.861071:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832647424/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763660.861087:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763660.861094:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21385)
00000800:00000200:3.1:1787763660.861381:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763660.861443:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787763660.861835:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763660.861837:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787763660.862016:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763660.862018:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000020:00000001:0.0:1787763661.500290:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763661.500294:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763661.500297:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763662.524314:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763662.524318:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763662.524320:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1787763663.548320:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763663.548324:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763663.548327:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763664.572354:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763664.572358:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763664.572362:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763665.596407:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763665.596411:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763665.596414:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763665.980252:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763665.980262:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763665.980266:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763665.980270:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763665.980275:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763665.980283:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21380
00000400:00000200:0.0:1787763665.980291:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763665.980297:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763665.980301:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763665.980304:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763665.980307:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763665.980311:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763665.980315:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763665.980320:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763665.980322:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763665.980326:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763665.980330:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763665.980333:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763665.980336:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763665.980339:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763665.980342:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763665.980344:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763665.980347:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763665.980351:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763665.980355:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763665.980358:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763665.980361:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763665.980366:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763665.980371:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763665.980375:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763665.980391:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763665.980401:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763665.980406:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763665.980413:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763665.980416:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763665.980423:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763665.980427:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763665.980432:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763665.980441:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763665.980448:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763665.980457:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832647552/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763665.980469:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763665.980474:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763665.980481:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832647552/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763665.980520:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763665.980523:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763665.980527:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763665.980528:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763665.980529:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763665.980549:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763665.980552:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21380
00000400:00000200:0.0:1787763665.980557:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763665.980560:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763665.980562:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763665.980565:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763665.980568:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763665.980570:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763665.980572:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763665.980575:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763665.980577:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763665.980579:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763665.980582:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763665.980585:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763665.980588:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763665.980591:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763665.980595:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763665.980599:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763665.980608:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763665.980610:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763665.980613:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763665.980616:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763665.980619:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763665.980624:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763665.980626:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763665.980628:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763665.980632:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763665.980635:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763665.980638:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763665.980643:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763665.980645:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763665.980648:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763665.980649:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763665.980651:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763665.980653:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763665.980655:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763665.980658:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787763665.980662:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763665.980667:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832647680/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763665.980676:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763665.980682:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763665.980687:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832647680/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763665.980696:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763665.980699:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763665.980703:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763665.980705:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763665.980706:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763665.980709:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763665.980711:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21380
00000400:00000200:0.0:1787763665.980715:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763665.980718:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763665.980720:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763665.980724:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763665.980726:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763665.980728:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763665.980730:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763665.980733:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763665.980734:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763665.980735:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763665.980738:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763665.980740:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763665.980742:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763665.980745:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763665.980747:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763665.980750:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763665.980752:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763665.980754:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763665.980757:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763665.980760:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763665.980761:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763665.980763:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763665.980766:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763665.980767:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763665.980769:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763665.980772:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763665.980773:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763665.980777:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763665.980779:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763665.980782:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763665.980783:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763665.980785:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763665.980786:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763665.980788:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763665.980792:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787763665.980796:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763665.980800:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832647808/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763665.980809:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763665.980811:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763665.980815:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832647808/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763665.980824:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763665.980827:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763665.980831:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763665.980834:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763665.980837:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763665.980838:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763665.980840:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763665.980842:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763665.980845:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21380
00000400:00000200:0.0:1787763665.980848:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763665.980852:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763665.980856:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763665.980862:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763665.980865:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763665.980871:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763665.980877:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763665.980881:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763665.980885:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763665.980889:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763665.980893:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763665.980896:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763665.980901:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763665.980909:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763665.980913:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763665.980919:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763665.980924:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763665.980928:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763665.980933:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763665.980939:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763665.980944:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763665.980949:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763665.980951:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763665.980954:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763665.980960:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763665.980964:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763665.980966:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763665.980974:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000020:00000040:0.0:1787763665.980978:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763665.980984:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763665.980986:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763665.980987:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763665.980990:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763665.980992:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763665.980998:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787763665.981002:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763665.981010:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832647936/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763665.981022:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763665.981027:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763665.981036:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832647936/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763665.981053:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763665.981059:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21390)
00000800:00000200:3.1:1787763665.981194:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763665.981197:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787763665.981638:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763665.981641:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787763665.981874:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763665.981876:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787763665.982054:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763665.982055:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000020:00000001:1.0:1787763666.620307:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763666.620311:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763666.620314:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763667.644379:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763667.644384:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763667.644387:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763668.668316:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763668.668339:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763668.668343:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763668.668349:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763668.668354:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763668.668358:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763668.668361:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763668.668364:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763668.668368:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763669.692260:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763669.692268:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763669.692274:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763669.692283:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763669.692292:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763669.692296:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763669.692300:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763669.692303:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763669.692307:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763669.692334:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763669.692338:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763669.692341:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763670.716211:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763670.716236:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763670.716239:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763670.716243:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763670.716247:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763670.716252:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763670.716256:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763670.716261:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763670.716266:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763670.716346:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763670.716349:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763670.716352:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787763671.100384:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763671.100392:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787763671.100394:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763671.100398:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763671.100402:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763671.100410:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21385
00000400:00000200:1.0:1787763671.100418:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763671.100425:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763671.100429:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763671.100433:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763671.100437:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763671.100439:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763671.100442:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763671.100446:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763671.100448:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787763671.100451:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763671.100455:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787763671.100458:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787763671.100460:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787763671.100464:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763671.100466:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763671.100469:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763671.100471:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763671.100474:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787763671.100477:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763671.100480:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763671.100483:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763671.100487:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763671.100490:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763671.100493:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763671.100509:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763671.100519:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932940.
00000020:00000040:1.0:1787763671.100523:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787763671.100527:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763671.100529:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763671.100532:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763671.100535:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763671.100539:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763671.100547:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8400.
00000100:00000001:1.0:1787763671.100553:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763671.100562:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932940 x1874599832648064/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763671.100574:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763671.100578:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763671.100583:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000af90eaa] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932940 x1874599832648064/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763671.100605:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763671.100609:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763671.100612:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787763671.100614:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763671.100616:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763671.100620:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763671.100624:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21385
00000400:00000200:1.0:1787763671.100630:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763671.100638:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763671.100643:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763671.100652:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763671.100660:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763671.100666:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763671.100672:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763671.100688:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763671.100694:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787763671.100698:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763671.100702:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787763671.100709:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787763671.100717:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787763671.100725:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763671.100730:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763671.100735:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763671.100742:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763671.100750:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787763671.100759:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763671.100765:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763671.100771:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763671.100779:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763671.100787:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763671.100790:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763671.100796:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763671.100803:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763671.100809:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763671.100817:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930780.
00000020:00000040:1.0:1787763671.100823:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763671.100830:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763671.100835:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763671.100840:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763671.100844:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763671.100849:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763671.100857:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9c00.
00000100:00000001:1.0:1787763671.100865:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763671.100877:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930780 x1874599832648192/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1787763671.100881:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763671.100887:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:1.0:1787763671.100900:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763671.100906:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763671.100916:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3977950] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930780 x1874599832648192/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763671.100934:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763671.100937:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763671.100945:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787763671.100949:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763671.100953:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763671.100958:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763671.100965:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21385
00000400:00000200:1.0:1787763671.100976:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763671.100984:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763671.100991:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763671.101000:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763671.101004:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763671.101009:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763671.101015:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763671.101020:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763671.101025:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787763671.101030:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763671.101037:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787763671.101042:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787763671.101049:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787763671.101058:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763671.101065:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763671.101072:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763671.101078:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763671.101084:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787763671.101093:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763671.101100:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763671.101106:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763671.101111:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763671.101116:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763671.101120:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763671.101125:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763671.101131:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763671.101155:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763671.101160:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932580.
00000020:00000040:1.0:1787763671.101167:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763671.101172:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763671.101177:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763671.101179:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763671.101182:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763671.101185:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763671.101192:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9400.
00000100:00000001:1.0:1787763671.101199:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763671.101209:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932580 x1874599832648320/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763671.101232:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763671.101239:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763671.101248:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d026924] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932580 x1874599832648320/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763671.101268:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763671.101274:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787763671.101281:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787763671.101287:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763671.101297:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787763671.101301:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763671.101305:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763671.101311:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763671.101319:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21385
00000400:00000200:1.0:1787763671.101326:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763671.101333:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763671.101338:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763671.101345:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763671.101350:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763671.101355:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763671.101358:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763671.101363:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763671.101367:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787763671.101372:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763671.101376:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787763671.101380:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787763671.101387:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:1.0:1787763671.101397:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763671.101401:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763671.101407:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763671.101412:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763671.101419:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:1.0:1787763671.101426:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763671.101432:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763671.101437:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763671.101443:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763671.101450:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763671.101455:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763671.101462:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1787763671.101467:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763671.101470:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763671.101478:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933840.
00000020:00000040:1.0:1787763671.101484:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787763671.101488:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763671.101492:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763671.101512:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763671.101516:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763671.101520:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763671.101528:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cac00.
00000100:00000001:1.0:1787763671.101536:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763671.101548:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933840 x1874599832648448/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763671.101571:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763671.101576:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763671.101584:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000083dd6d32] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933840 x1874599832648448/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.1:1787763671.101602:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763671.101606:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:1.0:1787763671.101608:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763671.101617:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21395)
00000800:00000200:3.1:1787763671.101815:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763671.101817:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787763671.102041:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763671.102043:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:0.0:1787763671.740222:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763671.740230:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763671.740234:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763671.740239:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763671.740245:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763671.740249:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763671.740256:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763671.740258:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763671.740261:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763672.764212:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763672.764226:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763672.764228:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763672.764232:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763672.764236:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763672.764239:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763672.764242:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763672.764244:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763672.764247:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763672.764299:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763672.764302:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763672.764305:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763673.788252:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763673.788261:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763673.788264:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763673.788271:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763673.788277:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763673.788282:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763673.788286:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763673.788292:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763673.788302:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763673.788361:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763673.788365:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763673.788368:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763674.812258:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763674.812270:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763674.812275:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763674.812286:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763674.812298:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763674.812309:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763674.812322:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763674.812330:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763674.812335:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763675.836321:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763675.836332:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763675.836336:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763675.836343:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763675.836348:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763675.836354:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763675.836357:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763675.836361:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763675.836365:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763675.836404:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763675.836411:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763675.836416:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763676.220286:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763676.220296:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763676.220300:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763676.220305:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763676.220311:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763676.220319:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21390
00000400:00000200:0.0:1787763676.220329:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763676.220336:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763676.220340:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763676.220344:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763676.220348:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763676.220351:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763676.220354:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763676.220359:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763676.220361:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763676.220365:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763676.220369:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763676.220372:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763676.220375:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763676.220378:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763676.220381:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763676.220400:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763676.220402:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763676.220408:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763676.220414:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763676.220421:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763676.220426:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763676.220432:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763676.220438:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763676.220441:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763676.220446:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763676.220459:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000020:00000040:0.0:1787763676.220487:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763676.220493:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763676.220497:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763676.220502:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763676.220506:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763676.220530:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763676.220543:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763676.220553:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763676.220567:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832648576/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763676.220580:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763676.220584:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763676.220590:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832648576/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763676.220618:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763676.220622:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763676.220626:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763676.220628:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763676.220630:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763676.220634:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763676.220637:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21390
00000400:00000200:0.0:1787763676.220642:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763676.220647:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763676.220650:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763676.220654:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763676.220659:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763676.220663:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763676.220666:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763676.220670:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763676.220673:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763676.220680:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763676.220683:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763676.220690:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763676.220695:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763676.220701:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763676.220704:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763676.220710:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763676.220714:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763676.220721:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763676.220728:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763676.220737:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763676.220743:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763676.220753:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763676.220764:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763676.220768:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763676.220777:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763676.220785:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763676.220794:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763676.220802:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
00000020:00000040:0.0:1787763676.220810:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763676.220816:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763676.220822:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763676.220827:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763676.220831:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763676.220837:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763676.220846:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763676.220856:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763676.220869:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832648704/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763676.220887:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763676.220892:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763676.220902:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832648704/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763676.220928:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763676.220935:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763676.220948:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763676.220954:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763676.220958:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763676.220966:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763676.220973:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21390
00000400:00000200:0.0:1787763676.220985:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763676.220993:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763676.220999:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763676.221009:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763676.221012:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763676.221019:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763676.221026:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763676.221029:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763676.221030:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763676.221032:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763676.221037:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763676.221043:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763676.221048:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763676.221055:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763676.221063:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763676.221067:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763676.221073:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763676.221078:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763676.221087:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763676.221092:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763676.221095:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763676.221103:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763676.221107:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763676.221112:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763676.221117:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763676.221127:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763676.221182:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763676.221194:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000020:00000040:0.0:1787763676.221203:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763676.221212:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763676.221216:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763676.221220:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763676.221222:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763676.221226:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763676.221236:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763676.221243:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763676.221256:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832648832/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1787763676.221265:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763676.221277:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:0.0:1787763676.221283:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763676.221291:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763676.221301:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832648832/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763676.221319:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763676.221326:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763676.221333:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763676.221340:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763676.221347:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763676.221353:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763676.221357:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763676.221364:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763676.221373:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21390
00000400:00000200:0.0:1787763676.221384:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763676.221393:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763676.221399:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763676.221410:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763676.221417:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763676.221423:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763676.221431:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763676.221441:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763676.221445:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763676.221452:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763676.221480:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763676.221491:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763676.221500:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763676.221514:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763676.221522:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763676.221532:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763676.221543:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763676.221550:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763676.221557:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763676.221565:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763676.221571:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763676.221584:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763676.221592:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763676.221599:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763676.221604:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763676.221612:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763676.221617:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763676.221628:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763676.221651:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763676.221657:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763676.221662:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763676.221667:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763676.221669:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763676.221673:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763676.221680:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763676.221688:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763676.221700:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832648960/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763676.221714:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763676.221720:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763676.221731:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832648960/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763676.221774:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763676.221781:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21400)
00000800:00000200:3.1:1787763676.222182:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763676.222185:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787763676.222620:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763676.222623:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787763676.222971:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763676.222974:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:0.0:1787763676.860247:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763676.860258:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763676.860262:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763676.860270:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763676.860278:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763676.860283:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763676.860287:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763676.860292:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763676.860296:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763676.860410:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763676.860414:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763676.860419:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763677.884242:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763677.884253:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763677.884256:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763677.884265:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763677.884271:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763677.884276:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763677.884281:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763677.884286:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763677.884292:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763678.908261:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763678.908276:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763678.908280:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763678.908291:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763678.908298:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763678.908305:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763678.908311:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763678.908316:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763678.908327:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763678.908421:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763678.908426:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763678.908430:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763679.932229:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763679.932236:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763679.932238:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763679.932243:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763679.932247:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763679.932250:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763679.932259:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763679.932266:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763679.932270:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763679.932318:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763679.932321:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763679.932323:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763680.956221:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763680.956232:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763680.956237:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763680.956245:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763680.956253:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763680.956258:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763680.956263:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763680.956267:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763680.956272:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763681.340232:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763681.340247:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763681.340250:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763681.340254:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763681.340260:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763681.340270:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21395
00000400:00000200:0.0:1787763681.340280:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763681.340292:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763681.340296:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763681.340300:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763681.340304:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763681.340307:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763681.340310:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763681.340314:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763681.340319:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763681.340322:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763681.340328:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763681.340333:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763681.340337:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763681.340342:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763681.340345:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763681.340348:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763681.340352:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763681.340355:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763681.340360:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763681.340362:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763681.340365:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763681.340369:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763681.340372:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763681.340374:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763681.340376:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763681.340387:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763681.340392:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763681.340397:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763681.340399:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763681.340402:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763681.340405:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763681.340412:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763681.340421:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787763681.340444:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763681.340456:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832649088/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763681.340465:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763681.340470:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763681.340475:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832649088/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763681.340495:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763681.340498:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763681.340501:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763681.340502:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763681.340503:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763681.340506:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763681.340508:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21395
00000400:00000200:0.0:1787763681.340511:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763681.340514:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763681.340516:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763681.340518:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763681.340521:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763681.340522:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763681.340525:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763681.340529:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763681.340530:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763681.340535:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763681.340539:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763681.340543:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763681.340545:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763681.340553:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763681.340558:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763681.340562:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763681.340565:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763681.340569:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763681.340574:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763681.340580:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763681.340584:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763681.340591:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763681.340596:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763681.340598:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763681.340604:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763681.340609:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763681.340611:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763681.340615:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763681.340619:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763681.340622:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763681.340623:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763681.340624:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763681.340627:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763681.340631:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763681.340636:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787763681.340641:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763681.340647:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832649216/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763681.340654:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763681.340658:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763681.340663:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832649216/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763681.340703:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763681.340707:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763681.340713:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763681.340716:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763681.340719:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763681.340726:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763681.340729:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21395
00000400:00000200:0.0:1787763681.340737:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763681.340740:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763681.340741:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763681.340743:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763681.340746:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763681.340751:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763681.340754:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763681.340756:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763681.340759:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763681.340761:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763681.340765:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763681.340767:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763681.340770:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763681.340774:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763681.340778:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763681.340783:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763681.340787:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763681.340791:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763681.340797:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763681.340802:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763681.340807:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763681.340810:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763681.340814:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763681.340817:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763681.340820:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763681.340824:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763681.340825:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763681.340830:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763681.340833:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763681.340835:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763681.340837:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763681.340841:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763681.340842:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763681.340843:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763681.340847:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787763681.340853:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763681.340860:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832649344/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1787763681.340867:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763681.340874:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:3.1:1787763681.340874:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000100:00000001:0.0:1787763681.340879:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763681.340886:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832649344/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763681.340896:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763681.340899:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763681.340904:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763681.340910:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763681.340914:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763681.340918:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763681.340921:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763681.340924:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763681.340929:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21395
00000400:00000200:0.0:1787763681.340937:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763681.340943:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763681.340948:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763681.340951:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763681.340954:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763681.340959:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763681.340962:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763681.340968:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763681.340970:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763681.340974:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763681.340977:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763681.340982:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763681.340985:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763681.340990:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763681.340992:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763681.340995:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763681.340998:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763681.341003:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763681.341009:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763681.341011:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763681.341015:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763681.341018:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763681.341019:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763681.341021:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763681.341023:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763681.341027:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763681.341028:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763681.341031:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763681.341033:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763681.341035:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763681.341040:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763681.341042:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763681.341043:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763681.341044:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763681.341047:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763681.341050:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763681.341058:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832649472/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763681.341067:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763681.341071:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763681.341079:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832649472/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763681.341088:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763681.341092:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21406)
00000800:00000200:3.1:1787763681.341390:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763681.341393:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787763681.341648:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763681.341650:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787763681.341839:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763681.341840:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000100:00000001:0.0:1787763681.980215:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763681.980236:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763681.980238:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763681.980245:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763681.980249:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763681.980253:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763681.980256:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763681.980258:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763681.980261:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763681.980318:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763681.980322:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763681.980326:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763683.004257:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763683.004274:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763683.004276:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763683.004284:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763683.004289:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763683.004293:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763683.004297:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763683.004301:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763683.004306:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763683.004362:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763683.004365:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763683.004368:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763684.028291:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763684.028301:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763684.028304:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763684.028314:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763684.028320:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763684.028327:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763684.028333:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763684.028339:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763684.028348:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763685.052267:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763685.052277:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763685.052280:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763685.052285:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763685.052291:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763685.052296:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763685.052300:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763685.052304:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763685.052309:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763686.076270:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763686.076280:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763686.076285:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763686.076311:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763686.076315:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763686.076320:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763686.076324:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763686.076327:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763686.076334:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763686.076415:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763686.076418:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763686.076422:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787763686.461430:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763686.461440:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787763686.462465:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763686.462468:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787763686.462868:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763686.462872:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787763686.463321:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763686.463324:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000100:00000001:0.0:1787763687.100314:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763687.100324:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763687.100329:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763687.100341:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763687.100349:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763687.100357:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763687.100369:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763687.100373:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763687.100377:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763688.124271:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763688.124282:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763688.124288:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763688.124297:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763688.124302:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763688.124311:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763688.124322:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763688.124326:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763688.124331:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763689.148206:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763689.148213:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763689.148218:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763689.148225:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763689.148230:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763689.148235:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763689.148241:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763689.148244:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763689.148246:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763689.148280:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763689.148284:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763689.148287:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763690.172283:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763690.172295:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763690.172302:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763690.172311:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763690.172321:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763690.172328:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763690.172333:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763690.172337:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763690.172342:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763691.196257:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763691.196269:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763691.196277:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763691.196290:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763691.196301:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763691.196309:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763691.196317:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763691.196324:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763691.196331:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763691.580635:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763691.580859:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763691.580989:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763691.581076:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763692.220215:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763692.220226:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763692.220231:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763692.220253:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763692.220260:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763692.220268:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763692.220276:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763692.220282:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763692.220287:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763692.220330:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763692.220336:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763692.220340:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763693.244266:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763693.244275:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763693.244278:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763693.244285:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763693.244291:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763693.244296:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763693.244299:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763693.244306:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763693.244311:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763694.268321:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763694.268327:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763694.268330:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763694.268349:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763694.268354:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763694.268359:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763694.268363:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763694.268369:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763694.268373:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:0.0:1787763694.652540:0:257565:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.110:988
00000400:00000100:0.0:1787763694.652585:0:257565:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.110@tcp at host 192.168.203.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:0.0:1787763694.652595:0:257565:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763694.652599:0:257565:0:(router.c:1791:lnet_notify()) 192.168.203.10@tcp notifying 192.168.203.110@tcp: down
00000400:00000200:0.0:1787763694.652604:0:257565:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763694.652609:0:257565:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:0.0:1787763694.652614:0:257565:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:0.0:1787763694.652618:0:257565:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186f00.
00000400:00000200:0.0:1787763694.652624:0:257565:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1787763694.652629:0:257565:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1787763694.652635:0:257565:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32220
00000400:00000010:0.0:1787763694.652639:0:257565:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32220.
00000100:00000001:0.0:1787763694.652645:0:257565:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763694.652655:0:257565:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f865ad00 x1874599832650112/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763694.652679:0:257565:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763694.652683:0:257565:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832650112/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763716 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763694.652702:0:257565:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763694.652704:0:257565:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:0.0:1787763694.652713:0:257565:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:0.0:1787763694.652716:0:257565:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186100.
00000400:00000200:0.0:1787763694.652721:0:257565:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1787763694.652724:0:257565:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1787763694.652728:0:257565:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32cc0
00000400:00000010:0.0:1787763694.652730:0:257565:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32cc0.
00000100:00000001:0.0:1787763694.652733:0:257565:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763694.652738:0:257565:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f8658780 x1874599832650240/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763694.652749:0:257565:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763694.652756:0:257565:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832650240/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763716 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763694.652766:0:257565:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763694.652777:0:257565:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:0.0:1787763694.652785:0:257565:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:0.0:1787763694.652791:0:257565:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186000.
00000400:00000200:0.0:1787763694.652795:0:257565:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1787763694.652803:0:257565:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1787763694.652809:0:257565:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32908
00000400:00000010:0.0:1787763694.652814:0:257565:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32908.
00000100:00000001:0.0:1787763694.652822:0:257565:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763694.652838:0:257565:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f8659a40 x1874599832650368/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763694.652876:0:257565:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763694.652887:0:257565:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832650368/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763716 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763694.652909:0:257565:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763694.652913:0:257565:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:0.0:1787763694.652940:0:257565:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:0.0:1787763694.652948:0:257565:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186e00.
00000400:00000200:0.0:1787763694.652958:0:257565:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1787763694.652971:0:257565:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1787763694.652981:0:257565:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d322a8
00000400:00000010:0.0:1787763694.652987:0:257565:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d322a8.
00000100:00000001:0.0:1787763694.652994:0:257565:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763694.653007:0:257565:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f8658000 x1874599832650496/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763694.653030:0:257565:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763694.653040:0:257565:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832650496/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763716 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763694.653053:0:257565:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763694.653059:0:257565:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763695.292252:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763695.292265:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763695.292274:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763695.292290:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763695.292303:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763695.292313:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763695.292324:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763695.292355:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763695.292365:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763695.292455:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763695.292461:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763695.292466:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763696.316266:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763696.316293:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763696.316298:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763696.316306:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763696.316316:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763696.316355:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763696.316362:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763696.316367:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763696.316372:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763696.701432:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763696.701447:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787763696.702658:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763696.702660:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787763696.702870:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763696.702872:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787763696.703095:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763696.703097:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:0.0:1787763697.340279:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763697.340300:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763697.340304:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763697.340310:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763697.340334:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763697.340342:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763697.340347:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763697.340368:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763697.340373:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763698.364488:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763698.364494:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763698.364499:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763699.388414:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763699.388419:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763699.388422:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1787763701.436308:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763701.436314:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763701.436318:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000800:00000200:3.1:1787763701.821222:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763701.821232:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787763701.822460:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763701.822463:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787763701.822721:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763701.822723:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787763701.823006:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763701.823008:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:0.0:1787763702.460458:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763702.460467:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763702.460473:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1787763703.484488:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763703.484494:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763703.484499:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1787763704.508447:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763704.508454:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763704.508459:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763705.532463:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763705.532468:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763705.532471:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1787763706.556330:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763706.556335:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763706.556340:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763706.940307:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763706.940315:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763706.940320:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763706.940324:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763706.940328:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763706.940337:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21421
00000400:00000200:0.0:1787763706.940346:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763706.940353:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763706.940356:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763706.940358:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763706.940362:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763706.940364:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763706.940367:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763706.940371:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763706.940373:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763706.940376:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763706.940380:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763706.940382:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763706.940384:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763706.940386:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763706.940387:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763706.940388:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763706.940390:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763706.940392:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763706.940395:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763706.940397:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763706.940400:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763706.940404:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763706.940406:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763706.940408:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763706.940410:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763706.940423:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000020:00000040:0.0:1787763706.940426:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763706.940430:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763706.940432:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763706.940435:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763706.940438:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763706.940443:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763706.940454:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763706.940462:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763706.940472:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832651648/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763706.940482:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763706.940487:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763706.940492:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832651648/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763706.940516:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763706.940518:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763706.940521:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763706.940523:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763706.940524:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763706.940526:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763706.940528:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21421
00000400:00000200:0.0:1787763706.940530:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763706.940532:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763706.940534:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763706.940537:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763706.940539:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763706.940541:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763706.940543:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763706.940544:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763706.940545:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763706.940547:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763706.940548:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763706.940550:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763706.940551:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763706.940555:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763706.940559:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763706.940562:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763706.940566:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763706.940567:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763706.940570:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763706.940573:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763706.940576:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763706.940584:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763706.940589:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763706.940593:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763706.940599:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763706.940605:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763706.940608:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763706.940612:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787763706.940615:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763706.940619:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763706.940620:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763706.940621:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763706.940625:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763706.940627:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763706.940633:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763706.940637:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763706.940642:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832651776/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763706.940653:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763706.940655:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763706.940661:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832651776/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763706.940669:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763706.940674:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763706.940677:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763706.940679:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763706.940680:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763706.940686:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763706.940691:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21421
00000400:00000200:0.0:1787763706.940697:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763706.940701:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763706.940705:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763706.940712:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763706.940716:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763706.940721:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763706.940724:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763706.940729:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763706.940730:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763706.940734:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763706.940736:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763706.940737:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763706.940739:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763706.940746:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763706.940749:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763706.940751:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763706.940758:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763706.940763:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763706.940770:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763706.940777:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763706.940783:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763706.940789:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763706.940795:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763706.940800:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763706.940803:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763706.940809:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763706.940810:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763706.940814:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
00000020:00000040:0.0:1787763706.940818:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763706.940824:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763706.940828:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763706.940834:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763706.940838:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763706.940843:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763706.940851:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787763706.940859:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763706.940860:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763706.940866:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000100:00100000:0.0:1787763706.940870:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832651904/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763706.940884:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763706.940886:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763706.940893:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832651904/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763706.940905:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763706.940908:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763706.940913:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763706.940917:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763706.940920:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763706.940924:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763706.940926:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763706.940931:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763706.940935:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21421
00000400:00000200:0.0:1787763706.940943:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763706.940950:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763706.940955:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763706.940961:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763706.940963:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763706.940966:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763706.940968:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763706.940971:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763706.940973:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763706.940975:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763706.940979:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763706.940983:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763706.940986:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763706.940990:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763706.940993:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763706.940994:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763706.940995:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763706.940998:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763706.941003:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763706.941005:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763706.941006:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763706.941008:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763706.941012:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763706.941015:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763706.941020:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763706.941026:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763706.941030:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763706.941038:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763706.941045:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763706.941050:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763706.941055:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763706.941061:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763706.941065:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763706.941070:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763706.941077:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763706.941083:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763706.941092:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832652032/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763706.941106:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763706.941110:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763706.941117:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832652032/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763706.941129:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763706.941159:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21431)
00000800:00000200:3.1:1787763706.941408:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763706.941410:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787763706.941614:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763706.941616:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787763706.941804:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763706.941806:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:1.0:1787763707.580371:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763707.580380:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763707.580385:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763708.604477:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763708.604484:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763708.604489:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1787763709.628468:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763709.628474:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763709.628479:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1787763710.652345:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763710.652348:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763710.652351:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763711.676348:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763711.676351:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763711.676354:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763712.060246:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763712.060255:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763712.060258:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763712.060262:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763712.060268:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763712.060273:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21426
00000400:00000200:0.0:1787763712.060280:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763712.060287:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763712.060290:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763712.060293:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763712.060296:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763712.060298:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763712.060300:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763712.060303:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763712.060304:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763712.060306:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763712.060309:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763712.060311:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763712.060312:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763712.060315:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763712.060316:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763712.060317:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763712.060319:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763712.060321:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763712.060324:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763712.060326:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763712.060329:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763712.060331:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763712.060333:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763712.060335:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763712.060337:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763712.060344:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763712.060347:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763712.060350:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763712.060351:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763712.060354:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763712.060356:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763712.060359:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763712.060365:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787763712.060370:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763712.060377:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832652160/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763712.060385:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763712.060389:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763712.060394:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832652160/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763712.060415:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763712.060419:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763712.060423:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763712.060425:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763712.060428:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763712.060432:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763712.060436:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21426
00000400:00000200:0.0:1787763712.060445:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763712.060453:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763712.060458:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763712.060465:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763712.060471:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763712.060476:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763712.060483:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763712.060490:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763712.060494:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763712.060499:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763712.060504:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763712.060510:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763712.060516:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763712.060524:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763712.060528:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763712.060531:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763712.060534:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763712.060538:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763712.060541:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763712.060547:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763712.060548:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763712.060553:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763712.060558:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763712.060560:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763712.060564:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763712.060570:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763712.060574:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763712.060580:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763712.060584:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763712.060590:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763712.060592:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763712.060596:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763712.060599:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763712.060603:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763712.060609:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787763712.060616:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763712.060627:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832652288/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763712.060640:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763712.060642:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763712.060650:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832652288/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763712.060674:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763712.060682:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763712.060691:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763712.060697:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763712.060700:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763712.060704:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763712.060709:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21426
00000400:00000200:0.0:1787763712.060718:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763712.060725:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763712.060729:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000200:3.1:1787763712.060730:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763712.060736:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:3.1:1787763712.060736:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000100:00000001:0.0:1787763712.060742:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763712.060749:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763712.060756:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763712.060762:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763712.060764:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763712.060766:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763712.060770:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763712.060776:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763712.060782:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763712.060791:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763712.060797:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763712.060803:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763712.060808:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763712.060811:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763712.060817:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763712.060823:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763712.060827:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763712.060831:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763712.060838:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763712.060842:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763712.060846:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763712.060849:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763712.060854:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763712.060859:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763712.060863:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763712.060865:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763712.060868:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763712.060870:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763712.060872:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763712.060876:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763712.060882:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787763712.060887:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763712.060895:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832652416/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763712.060908:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763712.060913:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763712.060931:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832652416/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763712.060947:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763712.060952:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763712.060961:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763712.060968:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763712.060973:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763712.060977:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763712.060982:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763712.060986:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763712.060991:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21426
00000400:00000200:0.0:1787763712.060998:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763712.061005:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763712.061011:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763712.061018:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763712.061021:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763712.061025:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763712.061030:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763712.061036:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763712.061040:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763712.061045:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763712.061049:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763712.061058:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763712.061063:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763712.061071:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763712.061075:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763712.061081:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763712.061086:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763712.061093:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763712.061099:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763712.061103:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763712.061107:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763712.061111:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763712.061116:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763712.061119:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763712.061124:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763712.061128:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763712.061130:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763712.061162:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763712.061166:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763712.061171:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763712.061174:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763712.061177:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763712.061181:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763712.061184:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763712.061189:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763712.061194:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763712.061203:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832652544/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763712.061229:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763712.061234:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763712.061242:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832652544/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763712.061255:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763712.061261:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21436)
00000800:00000200:3.1:1787763712.061333:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763712.061335:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787763712.061543:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763712.061546:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787763712.061794:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763712.061796:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:1.0:1787763712.700433:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763712.700442:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763712.700450:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1787763713.724553:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763713.724558:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763713.724561:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:0.0:1787763714.748307:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763714.748311:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763714.748313:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000020:00000001:1.0:1787763715.772305:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763715.772309:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763715.772312:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763716.796291:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763716.796299:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763716.796302:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763716.796307:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763716.796312:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763716.796316:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763716.796320:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763716.796323:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1787763716.796325:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000100:00000001:1.0:1787763716.796327:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763716.796335:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763716.796343:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763717.180233:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763717.180239:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763717.180242:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763717.180245:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763717.180249:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763717.180255:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21431
00000400:00000200:0.0:1787763717.180261:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763717.180266:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763717.180269:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763717.180271:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763717.180273:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763717.180275:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763717.180277:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763717.180280:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763717.180281:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763717.180283:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763717.180286:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763717.180288:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763717.180289:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763717.180291:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763717.180293:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763717.180294:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763717.180295:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763717.180297:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763717.180299:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763717.180301:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763717.180303:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763717.180305:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763717.180307:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763717.180309:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763717.180310:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763717.180318:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000020:00000040:0.0:1787763717.180320:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763717.180323:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763717.180324:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763717.180326:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763717.180328:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763717.180331:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763717.180338:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787763717.180343:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763717.180349:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832652672/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763717.180357:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763717.180361:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763717.180364:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832652672/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763717.180383:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763717.180386:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763717.180390:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763717.180391:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763717.180393:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763717.180396:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763717.180399:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21431
00000400:00000200:0.0:1787763717.180402:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763717.180405:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763717.180408:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763717.180410:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763717.180413:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763717.180415:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763717.180417:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763717.180419:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763717.180420:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763717.180422:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763717.180424:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763717.180426:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763717.180428:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763717.180431:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763717.180433:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763717.180435:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763717.180438:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763717.180440:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763717.180443:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763717.180446:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763717.180448:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763717.180453:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763717.180455:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763717.180457:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763717.180461:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763717.180464:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763717.180466:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763717.180470:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
00000020:00000040:0.0:1787763717.180472:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763717.180475:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763717.180476:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763717.180478:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763717.180480:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763717.180481:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763717.180485:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787763717.180488:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763717.180493:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832652800/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763717.180500:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763717.180503:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763717.180506:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832652800/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763717.180515:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763717.180518:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763717.180521:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763717.180523:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763717.180525:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763717.180528:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763717.180530:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21431
00000400:00000200:0.0:1787763717.180534:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763717.180537:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763717.180539:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763717.180542:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763717.180544:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763717.180545:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763717.180547:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763717.180550:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763717.180551:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763717.180552:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763717.180555:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763717.180557:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763717.180559:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763717.180562:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763717.180564:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763717.180566:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763717.180568:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763717.180569:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763717.180572:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763717.180575:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763717.180577:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763717.180579:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763717.180581:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763717.180583:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763717.180585:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763717.180587:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763717.180590:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763717.180593:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000020:00000040:0.0:1787763717.180596:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763717.180598:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763717.180599:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763717.180601:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763717.180602:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763717.180604:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763717.180607:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787763717.180610:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763717.180620:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00100000:0.0:1787763717.180623:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832652928/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1787763717.180624:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:0.0:1787763717.180631:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763717.180633:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763717.180636:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832652928/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763717.180645:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763717.180647:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763717.180651:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763717.180653:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763717.180657:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763717.180658:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763717.180660:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763717.180663:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763717.180666:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21431
00000400:00000200:0.0:1787763717.180669:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763717.180673:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763717.180675:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763717.180678:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763717.180680:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763717.180683:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763717.180685:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763717.180687:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763717.180688:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763717.180690:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763717.180692:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763717.180695:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763717.180697:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763717.180699:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763717.180702:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763717.180704:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763717.180705:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763717.180707:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763717.180710:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763717.180712:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763717.180714:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763717.180716:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763717.180719:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763717.180721:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763717.180724:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763717.180726:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763717.180728:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763717.180733:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763717.180735:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763717.180738:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763717.180739:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763717.180741:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763717.180742:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763717.180744:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763717.180748:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787763717.180750:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763717.180755:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832653056/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763717.180764:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763717.180766:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763717.180770:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832653056/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763717.180779:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763717.180783:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21441)
00000800:00000200:3.1:1787763717.180969:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763717.180972:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787763717.181201:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763717.181203:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787763717.181389:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763717.181390:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:1.0:1787763717.820272:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763717.820284:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763717.820287:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763717.820294:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763717.820301:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763717.820310:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763717.820321:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763717.820325:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763717.820330:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763717.820419:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763717.820426:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763717.820432:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763718.844279:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763718.844291:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763718.844298:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763718.844305:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763718.844318:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763718.844327:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763718.844334:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763718.844338:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763718.844341:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763719.868219:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763719.868227:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763719.868230:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763719.868234:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763719.868239:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763719.868243:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763719.868248:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763719.868251:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763719.868253:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763719.868284:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763719.868288:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763719.868291:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763720.892260:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763720.892269:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763720.892274:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763720.892284:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763720.892291:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763720.892296:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763720.892302:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763720.892306:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763720.892312:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763720.892393:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763720.892399:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763720.892404:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763721.916227:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763721.916234:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763721.916238:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763721.916242:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763721.916249:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763721.916253:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763721.916257:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763721.916259:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763721.916261:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763722.300297:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763722.300308:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763722.300315:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763722.300324:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763722.300330:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763722.300334:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763722.300339:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763722.300345:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763722.300349:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763722.300629:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763722.300640:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763722.300648:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763722.300668:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bde00 x1874599832653184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.300710:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763722.300719:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763722.300731:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763722.300754:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763722.300766:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763722.300780:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832653184:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763722.300797:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763722.300806:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763722.300815:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763722.300826:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763722.300841:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763722.300849:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763722.300866:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763722.300876:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763722.300890:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763722.300905:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832653184, portal 25
00000100:00000001:1.0:1787763722.300928:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763722.300933:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049853440 : -115096659698176 : ffff9751f87bde00)
00000100:00000040:1.0:1787763722.300952:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bde00 x1874599832653184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763747 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.300983:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763722.300998:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763722.301005:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763722.301015:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832653184, offset 0
00000400:00000200:1.0:1787763722.301035:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763722.301054:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763722.301068:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763722.301082:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763722.301095:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763722.301108:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763722.301119:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763722.301131:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58338:58338:256:4294967295] 192.168.203.110@tcp LPNI seq info [58338:58338:8:4294967295]
00000400:00000200:1.0:1787763722.301190:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763722.301213:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763722.301231:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763722.301247:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763722.301264:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763722.301273:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787763722.301288:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763722.301299:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1787763722.301311:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763722.301319:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763722.301330:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763722.301345:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763722.301355:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763722.301362:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763722.301373:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763722.301386:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bde00 x1874599832653184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763747 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.301416:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763722.301426:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bde00 x1874599832653184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763747 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.301450:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763722.301456:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763722.301467:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763722.301476:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763722.301482:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316110580
00000100:00000010:1.0:1787763722.301494:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4a20.
00000100:00000001:1.0:1787763722.301508:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787763722.301523:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316110580
00000100:00000010:1.0:1787763722.301532:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4cc0.
00000100:00000001:1.0:1787763722.301540:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763722.301549:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763722.301557:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763722.301563:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763722.301574:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763722/real 1787763722]  req@ffff9751f87bde00 x1874599832653184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763722.301595:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763722.301600:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763722.301606:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763722.301614:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763722.301620:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763722.301628:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763722.301636:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bde00 x1874599832653184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763722.301659:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bde00 x1874599832653184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.301675:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763722.301681:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.301688:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763722.301692:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763722.301704:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bde00 x1874599832653184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.301725:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763722.301736:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bde00 x1874599832653184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.301762:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763722.301776:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.301784:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763722.301791:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.301800:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763722.301807:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763722.301821:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763722.301835:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763722.301846:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763722.301867:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bde00 x1874599832653184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763722.301892:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832653184:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763722.301902:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.301906:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763722.301912:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bde00 x1874599832653184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.301920:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763722.301924:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763722.301927:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763722.301931:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763722.301934:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763722.301936:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763722.301938:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763722.301941:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612400.
02000000:00000001:1.0:1787763722.301946:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763722.301949:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763722.301951:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bde00.
00000100:00000001:1.0:1787763722.301955:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763722.301956:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.301959:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.301967:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763722.301969:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763722.301971:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763722.301973:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763722.301975:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763722.301979:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bed00 x1874599832653312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.301988:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763722.301990:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763722.301993:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763722.301995:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763722.301998:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763722.302001:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832653312:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763722.302005:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763722.302008:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763722.302010:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763722.302012:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763722.302017:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763722.302019:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763722.302024:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763722.302027:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763722.302032:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763722.302036:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832653312, portal 10
00000100:00000001:1.0:1787763722.302038:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763722.302040:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049857280 : -115096659694336 : ffff9751f87bed00)
00000100:00000040:1.0:1787763722.302048:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bed00 x1874599832653312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763747 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.302056:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763722.302059:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763722.302062:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763722.302065:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832653312, offset 0
00000400:00000200:1.0:1787763722.302070:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763722.302075:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763722.302080:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763722.302085:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763722.302088:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763722.302091:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763722.302094:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763722.302098:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58339:58339:256:4294967295] 192.168.203.110@tcp LPNI seq info [58339:58339:8:4294967295]
00000400:00000200:1.0:1787763722.302103:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763722.302111:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763722.302116:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763722.302120:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763722.302125:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763722.302129:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787763722.302186:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763722.302190:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1787763722.302195:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763722.302197:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763722.302201:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763722.302206:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763722.302210:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763722.302213:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763722.302216:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763722.302220:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bed00 x1874599832653312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763747 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.302231:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763722.302234:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bed00 x1874599832653312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763747 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.302242:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763722.302244:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763722.302249:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763722.302254:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763722.302258:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763722.302261:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316109345
00000100:00000001:1.0:1787763722.302264:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316110580
00000100:00000001:1.0:1787763722.302267:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316109345
00000100:00000001:1.0:1787763722.302270:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316110580
00000100:00000001:1.0:1787763722.302272:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763722.302275:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763722.302277:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763722.302281:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763722/real 1787763722]  req@ffff9751f87bed00 x1874599832653312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763722.302290:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763722.302292:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763722.302293:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763722.302296:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763722.302298:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763722.302300:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763722.302303:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bed00 x1874599832653312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763722.302313:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bed00 x1874599832653312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.302320:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763722.302322:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.302324:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763722.302326:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763722.302330:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bed00 x1874599832653312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.302337:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763722.302342:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bed00 x1874599832653312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.302351:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763722.302354:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.302356:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763722.302357:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.302360:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763722.302372:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763722.302375:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763722.302379:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763722.302381:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763722.302386:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bed00 x1874599832653312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763722.302395:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832653312:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763722.302400:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763722.302403:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87be1c0 x1874599832653440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.302411:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763722.302412:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763722.302415:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763722.302417:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763722.302419:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763722.302422:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832653440:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763722.302426:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763722.302428:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763722.302430:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763722.302432:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763722.302436:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787763722.302438:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763722.302443:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763722.302445:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763722.302449:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763722.302453:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832653440, portal 10
00000100:00000001:1.0:1787763722.302455:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763722.302456:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049854400 : -115096659697216 : ffff9751f87be1c0)
00000100:00000040:1.0:1787763722.302462:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87be1c0 x1874599832653440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763747 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.302470:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763722.302473:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763722.302475:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763722.302478:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832653440, offset 0
00000400:00000200:1.0:1787763722.302483:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763722.302487:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763722.302491:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763722.302495:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763722.302498:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763722.302501:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763722.302504:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763722.302507:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58340:58340:256:4294967295] 192.168.203.110@tcp LPNI seq info [58340:58340:8:4294967295]
00000400:00000200:1.0:1787763722.302512:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763722.302520:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763722.302524:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763722.302528:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763722.302532:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763722.302536:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787763722.302541:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763722.302543:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1787763722.302548:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763722.302550:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763722.302554:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763722.302558:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763722.302561:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763722.302563:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763722.302566:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763722.302570:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87be1c0 x1874599832653440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763747 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.302579:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763722.302583:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87be1c0 x1874599832653440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763747 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.302590:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763722.302592:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763722.302595:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763722.302598:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763722.302602:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763722.302604:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316109333
00000100:00000001:1.0:1787763722.302607:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316110580
00000100:00000001:1.0:1787763722.302610:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316109333
00000100:00000001:1.0:1787763722.302612:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316110580
00000100:00000001:1.0:1787763722.302614:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763722.302616:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763722.302618:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763722.302622:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763722/real 1787763722]  req@ffff9751f87be1c0 x1874599832653440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763722.302630:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763722.302632:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763722.302634:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763722.302636:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763722.302638:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763722.302640:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763722.302644:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87be1c0 x1874599832653440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763722.302653:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87be1c0 x1874599832653440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.302661:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763722.302663:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.302665:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763722.302666:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763722.302670:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87be1c0 x1874599832653440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.302678:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763722.302682:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87be1c0 x1874599832653440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.302689:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763722.302692:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.302694:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763722.302695:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.302697:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763722.302699:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763722.302702:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763722.302705:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763722.302707:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763722.302711:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87be1c0 x1874599832653440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763722.302720:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832653440:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763722.302724:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763722.302727:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc000 x1874599832653568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.302735:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763722.302737:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763722.302740:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763722.302742:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763722.302745:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763722.302748:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832653568:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763722.302752:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763722.302754:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763722.302755:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763722.302757:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763722.302761:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763722.302763:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763722.302767:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763722.302769:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763722.302773:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763722.302776:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832653568, portal 4
00000100:00000001:1.0:1787763722.302778:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763722.302780:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049845760 : -115096659705856 : ffff9751f87bc000)
00000100:00000040:1.0:1787763722.302786:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc000 x1874599832653568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763747 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.302793:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763722.302796:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763722.302798:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763722.302801:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832653568, offset 0
00000400:00000200:1.0:1787763722.302805:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763722.302808:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763722.302812:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763722.302815:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763722.302817:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763722.302820:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763722.302823:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763722.302827:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58341:58341:256:4294967295] 192.168.203.110@tcp LPNI seq info [58341:58341:8:4294967295]
00000400:00000200:1.0:1787763722.302832:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763722.302839:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763722.302845:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763722.302848:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763722.302852:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763722.302855:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787763722.302860:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763722.302863:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1787763722.302867:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763722.302869:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763722.302873:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763722.302877:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763722.302880:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763722.302882:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763722.302884:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763722.302888:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc000 x1874599832653568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763747 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.302897:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763722.302900:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc000 x1874599832653568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763747 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.302907:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763722.302909:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763722.302912:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763722.302915:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763722.302919:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763722.302921:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316109350
00000100:00000001:1.0:1787763722.302924:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316110580
00000100:00000001:1.0:1787763722.302927:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316109350
00000100:00000001:1.0:1787763722.302929:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316110580
00000100:00000001:1.0:1787763722.302931:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763722.302934:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763722.302936:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763722.302939:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763722/real 1787763722]  req@ffff9751f87bc000 x1874599832653568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763722.302949:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763722.302950:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763722.302953:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763722.302955:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763722.302957:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763722.302959:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763722.302962:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc000 x1874599832653568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763722.302971:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc000 x1874599832653568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.302978:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763722.302979:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.302981:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763722.302983:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763722.302987:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc000 x1874599832653568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.302994:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763722.302998:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc000 x1874599832653568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.303005:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763722.303008:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.303010:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763722.303011:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.303014:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763722.303015:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763722.303019:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763722.303022:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763722.303024:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763722.303029:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc000 x1874599832653568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763722.303038:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832653568:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763722.303042:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.303044:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763722.303048:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bed00 x1874599832653312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.303055:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763722.303057:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763722.303059:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763722.303061:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763722.303063:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763722.303064:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763722.303067:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763722.303069:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612c00.
02000000:00000001:1.0:1787763722.303071:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763722.303073:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763722.303074:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bed00.
00000100:00000001:1.0:1787763722.303077:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763722.303079:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.303081:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763722.303084:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87be1c0 x1874599832653440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.303091:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763722.303093:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763722.303094:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787763722.303097:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763722.303098:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763722.303099:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763722.303101:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763722.303103:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612800.
02000000:00000001:1.0:1787763722.303105:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763722.303107:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763722.303109:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87be1c0.
00000100:00000001:1.0:1787763722.303111:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763722.303113:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.303114:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763722.303118:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc000 x1874599832653568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763747 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763722.303125:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763722.303126:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763722.303128:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763722.303130:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763722.303131:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763722.303179:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763722.303182:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763722.303184:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613000.
02000000:00000001:1.0:1787763722.303187:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763722.303188:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763722.303190:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc000.
00000100:00000001:1.0:1787763722.303193:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763722.303194:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.303196:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.303200:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763722.303202:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763722.303204:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763722.303206:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763722.940296:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763722.940306:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763722.940312:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763722.940319:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763722.940327:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763722.940334:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.940340:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763722.940343:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763722.940346:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763722.940394:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763722.940400:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763722.940404:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763723.324257:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763723.324270:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763723.324275:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763723.324283:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763723.324291:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763723.324299:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763723.324304:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763723.324310:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763723.324316:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763723.964238:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763723.964246:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763723.964249:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763723.964256:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763723.964262:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763723.964266:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763723.964270:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763723.964272:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763723.964276:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763723.964394:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763723.964399:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763723.964403:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763724.348244:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763724.348252:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763724.348254:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763724.348261:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763724.348266:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763724.348270:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763724.348273:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763724.348275:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763724.348278:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763724.988288:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763724.988314:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763724.988317:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763724.988323:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763724.988328:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763724.988332:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763724.988336:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763724.988342:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763724.988350:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763725.372241:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763725.372247:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763725.372250:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763725.372254:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763725.372258:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763725.372261:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763725.372263:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763725.372265:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763725.372267:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763726.012242:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763726.012251:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763726.012255:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763726.012261:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763726.012267:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763726.012274:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763726.012279:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763726.012282:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763726.012287:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763726.012339:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763726.012344:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763726.012348:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763726.396274:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763726.396284:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763726.396287:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763726.396294:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763726.396310:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763726.396315:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763726.396318:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763726.396322:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763726.396325:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763727.036248:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763727.036263:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763727.036268:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763727.036276:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763727.036282:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763727.036288:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.036293:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763727.036297:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.036302:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763727.420234:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763727.420243:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763727.420248:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763727.420257:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763727.420265:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763727.420274:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763727.420281:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763727.420286:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763727.420292:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763727.420571:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763727.420590:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763727.420598:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763727.420631:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bf480 x1874599832653696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.420702:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763727.420715:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763727.420733:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763727.420745:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763727.420755:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763727.420767:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832653696:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763727.420779:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763727.420787:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763727.420796:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763727.420801:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763727.420819:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763727.420825:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763727.420845:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763727.420864:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763727.420876:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763727.420893:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832653696, portal 25
00000100:00000001:1.0:1787763727.420904:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763727.420913:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049859200 : -115096659692416 : ffff9751f87bf480)
00000100:00000040:1.0:1787763727.420937:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bf480 x1874599832653696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763752 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.420966:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763727.420976:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763727.420983:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763727.420994:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832653696, offset 0
00000400:00000200:1.0:1787763727.421013:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763727.421035:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763727.421056:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763727.421076:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763727.421091:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763727.421099:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763727.421107:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763727.421205:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58342:58342:256:4294967295] 192.168.203.110@tcp LPNI seq info [58342:58342:8:4294967295]
00000400:00000200:1.0:1787763727.421227:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763727.421257:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763727.421282:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763727.421296:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763727.421312:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763727.421323:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787763727.421343:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763727.421362:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1787763727.421383:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763727.421389:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763727.421400:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763727.421415:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763727.421438:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763727.421445:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763727.421462:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763727.421479:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bf480 x1874599832653696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763752 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.421513:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763727.421528:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bf480 x1874599832653696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763752 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.421572:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763727.421584:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763727.421595:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763727.421613:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763727.421619:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316115700
00000100:00000010:1.0:1787763727.421630:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4300.
00000100:00000001:1.0:1787763727.421642:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787763727.421656:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316115700
00000100:00000010:1.0:1787763727.421663:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4120.
00000100:00000001:1.0:1787763727.421668:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763727.421674:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763727.421680:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763727.421685:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763727.421697:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763727/real 1787763727]  req@ffff9751f87bf480 x1874599832653696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763727.421731:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763727.421735:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763727.421738:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763727.421742:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763727.421744:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763727.421748:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763727.421753:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bf480 x1874599832653696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763727.421764:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bf480 x1874599832653696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.421774:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763727.421776:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.421779:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763727.421781:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763727.421786:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bf480 x1874599832653696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.421794:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763727.421800:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bf480 x1874599832653696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.421809:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763727.421813:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.421816:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763727.421817:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.421821:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763727.421823:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763727.421827:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763727.421832:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763727.421835:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763727.421841:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bf480 x1874599832653696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763727.421852:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832653696:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763727.421858:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.421861:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763727.421865:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bf480 x1874599832653696/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.421873:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763727.421876:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763727.421880:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763727.421883:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763727.421887:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763727.421889:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763727.421892:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763727.421895:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613400.
02000000:00000001:1.0:1787763727.421900:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763727.421902:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763727.421905:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bf480.
00000100:00000001:1.0:1787763727.421909:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763727.421911:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.421914:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.421923:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763727.421925:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763727.421928:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763727.421930:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763727.421932:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763727.421937:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc000 x1874599832653824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.421947:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763727.421949:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763727.421952:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763727.421955:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763727.421957:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763727.421961:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832653824:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763727.421966:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763727.421969:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763727.421972:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763727.421974:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763727.421978:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763727.421981:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763727.421986:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763727.421989:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763727.421994:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763727.421998:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832653824, portal 10
00000100:00000001:1.0:1787763727.422001:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763727.422003:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049845760 : -115096659705856 : ffff9751f87bc000)
00000100:00000040:1.0:1787763727.422011:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc000 x1874599832653824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763752 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.422020:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763727.422023:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763727.422025:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763727.422028:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832653824, offset 0
00000400:00000200:1.0:1787763727.422034:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763727.422039:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763727.422044:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763727.422048:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763727.422052:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763727.422055:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763727.422059:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763727.422063:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58343:58343:256:4294967295] 192.168.203.110@tcp LPNI seq info [58343:58343:8:4294967295]
00000400:00000200:1.0:1787763727.422068:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763727.422077:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763727.422083:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763727.422087:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763727.422092:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763727.422096:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787763727.422103:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763727.422106:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1787763727.422164:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763727.422168:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763727.422174:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763727.422179:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763727.422185:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763727.422187:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763727.422190:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763727.422195:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc000 x1874599832653824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763752 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.422206:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763727.422210:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc000 x1874599832653824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763752 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.422219:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763727.422222:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763727.422227:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763727.422233:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763727.422238:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763727.422241:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316114465
00000100:00000001:1.0:1787763727.422245:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316115700
00000100:00000001:1.0:1787763727.422248:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316114465
00000100:00000001:1.0:1787763727.422250:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316115700
00000100:00000001:1.0:1787763727.422253:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763727.422255:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763727.422258:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763727.422262:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763727/real 1787763727]  req@ffff9751f87bc000 x1874599832653824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763727.422271:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763727.422273:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763727.422276:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763727.422279:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763727.422281:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763727.422284:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763727.422288:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc000 x1874599832653824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763727.422298:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc000 x1874599832653824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.422305:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763727.422307:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.422309:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763727.422311:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763727.422315:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc000 x1874599832653824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.422323:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763727.422328:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc000 x1874599832653824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.422336:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763727.422339:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.422341:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763727.422343:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.422345:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763727.422360:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763727.422363:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763727.422367:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763727.422370:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763727.422375:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc000 x1874599832653824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763727.422385:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832653824:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763727.422390:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763727.422395:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87be1c0 x1874599832653952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.422404:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763727.422405:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763727.422408:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763727.422411:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763727.422413:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763727.422417:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832653952:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763727.422421:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763727.422423:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763727.422426:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763727.422428:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763727.422433:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787763727.422435:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763727.422439:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763727.422442:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763727.422446:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763727.422450:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832653952, portal 10
00000100:00000001:1.0:1787763727.422452:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763727.422454:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049854400 : -115096659697216 : ffff9751f87be1c0)
00000100:00000040:1.0:1787763727.422461:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87be1c0 x1874599832653952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763752 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.422469:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763727.422472:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763727.422475:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763727.422478:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832653952, offset 0
00000400:00000200:1.0:1787763727.422482:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763727.422487:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763727.422491:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763727.422495:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763727.422499:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763727.422502:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763727.422505:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763727.422509:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58344:58344:256:4294967295] 192.168.203.110@tcp LPNI seq info [58344:58344:8:4294967295]
00000400:00000200:1.0:1787763727.422514:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763727.422522:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763727.422527:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763727.422531:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763727.422536:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763727.422539:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787763727.422544:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763727.422548:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1787763727.422552:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763727.422555:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763727.422559:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763727.422563:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763727.422566:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763727.422569:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763727.422572:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763727.422576:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87be1c0 x1874599832653952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763752 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.422586:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763727.422589:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87be1c0 x1874599832653952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763752 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.422598:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763727.422600:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763727.422603:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763727.422606:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763727.422611:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763727.422613:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316114453
00000100:00000001:1.0:1787763727.422616:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316115700
00000100:00000001:1.0:1787763727.422619:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316114453
00000100:00000001:1.0:1787763727.422624:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316115700
00000100:00000001:1.0:1787763727.422628:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763727.422633:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763727.422637:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763727.422644:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763727/real 1787763727]  req@ffff9751f87be1c0 x1874599832653952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763727.422660:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763727.422663:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763727.422666:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763727.422670:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763727.422674:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763727.422678:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763727.422686:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87be1c0 x1874599832653952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763727.422704:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87be1c0 x1874599832653952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.422719:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763727.422722:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.422726:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763727.422728:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763727.422735:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87be1c0 x1874599832653952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.422744:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763727.422749:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87be1c0 x1874599832653952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.422757:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763727.422760:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.422762:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763727.422763:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.422767:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763727.422769:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763727.422773:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763727.422777:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763727.422780:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763727.422785:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87be1c0 x1874599832653952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763727.422794:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832653952:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763727.422799:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763727.422803:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bed00 x1874599832654080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.422812:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763727.422814:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763727.422817:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763727.422819:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763727.422822:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763727.422826:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832654080:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763727.422829:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763727.422832:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763727.422834:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763727.422836:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763727.422841:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763727.422844:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763727.422849:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763727.422852:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763727.422856:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763727.422860:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832654080, portal 4
00000100:00000001:1.0:1787763727.422863:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763727.422865:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049857280 : -115096659694336 : ffff9751f87bed00)
00000100:00000040:1.0:1787763727.422872:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bed00 x1874599832654080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763752 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.422881:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763727.422884:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763727.422886:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763727.422889:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832654080, offset 0
00000400:00000200:1.0:1787763727.422894:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763727.422899:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763727.422904:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763727.422907:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763727.422911:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763727.422914:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763727.422917:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763727.422921:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58345:58345:256:4294967295] 192.168.203.110@tcp LPNI seq info [58345:58345:8:4294967295]
00000400:00000200:1.0:1787763727.422926:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763727.422933:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763727.422939:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763727.422942:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763727.422947:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763727.422951:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787763727.422956:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763727.422960:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1787763727.422964:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763727.422967:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763727.422971:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763727.422975:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763727.422978:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763727.422981:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763727.422984:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763727.422988:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bed00 x1874599832654080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763752 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.422997:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763727.423001:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bed00 x1874599832654080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763752 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.423018:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763727.423020:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763727.423024:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763727.423027:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763727.423031:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763727.423034:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316114470
00000100:00000001:1.0:1787763727.423037:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316115700
00000100:00000001:1.0:1787763727.423040:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316114470
00000100:00000001:1.0:1787763727.423043:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316115700
00000100:00000001:1.0:1787763727.423045:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763727.423048:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763727.423050:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763727.423054:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763727/real 1787763727]  req@ffff9751f87bed00 x1874599832654080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763727.423063:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763727.423064:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763727.423066:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763727.423069:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763727.423071:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763727.423073:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763727.423077:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bed00 x1874599832654080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763727.423087:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bed00 x1874599832654080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.423095:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763727.423096:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.423098:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763727.423100:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763727.423104:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bed00 x1874599832654080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.423167:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763727.423174:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bed00 x1874599832654080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.423183:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763727.423187:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.423189:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763727.423190:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.423193:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763727.423195:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763727.423198:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763727.423202:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763727.423205:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763727.423209:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bed00 x1874599832654080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763727.423219:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832654080:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763727.423223:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.423226:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763727.423230:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc000 x1874599832653824/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.423237:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763727.423239:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763727.423242:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763727.423245:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763727.423248:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763727.423250:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763727.423252:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763727.423255:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611000.
02000000:00000001:1.0:1787763727.423259:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763727.423260:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763727.423262:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc000.
00000100:00000001:1.0:1787763727.423266:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763727.423268:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.423270:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763727.423274:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87be1c0 x1874599832653952/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.423282:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763727.423283:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763727.423285:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787763727.423288:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763727.423289:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763727.423290:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763727.423293:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763727.423294:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610000.
02000000:00000001:1.0:1787763727.423297:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763727.423298:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763727.423300:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87be1c0.
00000100:00000001:1.0:1787763727.423302:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763727.423304:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.423306:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763727.423309:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bed00 x1874599832654080/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763752 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763727.423316:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763727.423318:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763727.423320:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787763727.423322:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763727.423323:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763727.423325:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763727.423327:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763727.423328:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611400.
02000000:00000001:1.0:1787763727.423331:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763727.423332:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763727.423334:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bed00.
00000100:00000001:1.0:1787763727.423336:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763727.423339:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.423341:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763727.423346:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763727.423348:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763727.423350:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763727.423352:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763728.060316:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763728.060326:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763728.060329:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763728.060339:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763728.060348:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763728.060352:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763728.060355:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763728.060358:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763728.060361:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763728.444226:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763728.444236:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763728.444238:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763728.444244:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763728.444248:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763728.444252:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763728.444255:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763728.444258:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763728.444261:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763729.084316:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763729.084321:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763729.084322:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00010000:0.0:1787763729.084330:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763729.084345:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763729.084350:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763729.084358:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763729.084381:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763729.084387:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763729.084392:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763729.084397:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763729.084402:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763729.468260:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763729.468271:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763729.468274:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763729.468281:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763729.468286:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763729.468292:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763729.468295:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763729.468299:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763729.468303:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763730.108302:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763730.108315:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763730.108322:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763730.108331:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763730.108338:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763730.108346:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763730.108354:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763730.108358:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763730.108366:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763730.492317:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763730.492331:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763730.492336:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763730.492345:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763730.492353:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763730.492360:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763730.492365:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763730.492369:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763730.492376:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763731.132268:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763731.132278:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763731.132284:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763731.132293:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763731.132302:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763731.132310:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763731.132318:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763731.132322:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763731.132329:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763731.516315:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763731.516327:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763731.516331:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763731.516339:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763731.516346:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763731.516352:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763731.516356:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763731.516361:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763731.516365:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763732.156292:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763732.156301:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763732.156305:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763732.156313:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763732.156320:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763732.156329:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.156335:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763732.156346:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.156351:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763732.156446:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763732.156453:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763732.156458:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763732.540254:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763732.540262:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763732.540267:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763732.540275:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763732.540282:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763732.540289:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763732.540294:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763732.540299:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763732.540305:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763732.540569:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763732.540578:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763732.540584:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763732.540605:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bde00 x1874599832654208/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.540619:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763732.540622:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763732.540631:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763732.540638:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763732.540643:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763732.540653:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832654208:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763732.540663:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763732.540673:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763732.540680:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763732.540685:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763732.540693:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787763732.540696:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763732.540704:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763732.540710:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763732.540719:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763732.540730:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832654208, portal 25
00000100:00000001:1.0:1787763732.540734:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763732.540737:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049853440 : -115096659698176 : ffff9751f87bde00)
00000100:00000040:1.0:1787763732.540745:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bde00 x1874599832654208/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763757 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.540756:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763732.540763:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763732.540769:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763732.540775:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832654208, offset 0
00000400:00000200:1.0:1787763732.540782:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763732.540792:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763732.540802:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763732.540811:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763732.540817:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763732.540822:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763732.540826:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763732.540831:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58346:58346:256:4294967295] 192.168.203.110@tcp LPNI seq info [58346:58346:8:4294967295]
00000400:00000200:1.0:1787763732.540843:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763732.540859:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763732.540868:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763732.540876:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763732.540886:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763732.540892:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787763732.540899:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763732.540904:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787763732.540910:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763732.540914:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763732.540921:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763732.540930:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763732.540939:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763732.540945:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763732.540953:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763732.540963:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bde00 x1874599832654208/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763757 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.540984:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763732.540994:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bde00 x1874599832654208/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763757 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.541008:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763732.541011:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763732.541025:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763732.541036:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763732.541042:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316120819
00000100:00000010:1.0:1787763732.541051:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4ea0.
00000100:00000001:1.0:1787763732.541065:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787763732.541095:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316120819
00000100:00000010:1.0:1787763732.541103:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4ba0.
00000100:00000001:1.0:1787763732.541111:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763732.541119:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763732.541125:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763732.541131:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763732.541176:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763732/real 1787763732]  req@ffff9751f87bde00 x1874599832654208/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763732.541191:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763732.541195:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763732.541199:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763732.541204:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763732.541211:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763732.541218:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763732.541227:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bde00 x1874599832654208/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763732.541246:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bde00 x1874599832654208/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.541262:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763732.541267:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.541273:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763732.541278:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763732.541285:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bde00 x1874599832654208/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.541292:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763732.541296:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bde00 x1874599832654208/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.541301:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763732.541303:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.541305:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763732.541305:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.541308:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763732.541309:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763732.541312:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763732.541317:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763732.541321:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763732.541326:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bde00 x1874599832654208/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763732.541337:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832654208:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763732.541343:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.541346:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763732.541351:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bde00 x1874599832654208/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.541359:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763732.541363:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763732.541366:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787763732.541370:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763732.541373:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763732.541374:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763732.541377:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763732.541380:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611400.
02000000:00000001:1.0:1787763732.541385:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763732.541389:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763732.541391:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bde00.
00000100:00000001:1.0:1787763732.541395:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763732.541396:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.541398:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.541410:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763732.541413:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763732.541415:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763732.541417:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763732.541419:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763732.541424:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bfc00 x1874599832654336/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.541432:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763732.541434:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763732.541438:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763732.541440:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763732.541442:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763732.541446:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832654336:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763732.541453:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763732.541460:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763732.541465:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763732.541470:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763732.541476:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787763732.541480:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763732.541489:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763732.541495:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763732.541501:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763732.541508:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832654336, portal 10
00000100:00000001:1.0:1787763732.541511:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763732.541515:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049861120 : -115096659690496 : ffff9751f87bfc00)
00000100:00000040:1.0:1787763732.541525:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bfc00 x1874599832654336/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763757 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.541543:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763732.541548:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763732.541552:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763732.541555:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832654336, offset 0
00000400:00000200:1.0:1787763732.541561:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763732.541569:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763732.541577:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763732.541581:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763732.541588:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763732.541594:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763732.541599:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763732.541606:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58347:58347:256:4294967295] 192.168.203.110@tcp LPNI seq info [58347:58347:8:4294967295]
00000400:00000200:1.0:1787763732.541615:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763732.541631:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763732.541642:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763732.541649:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763732.541664:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763732.541671:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787763732.541683:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763732.541689:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787763732.541698:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763732.541703:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763732.541710:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763732.541720:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763732.541729:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763732.541735:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763732.541741:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763732.541751:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bfc00 x1874599832654336/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763757 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.541773:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763732.541787:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bfc00 x1874599832654336/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763757 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.541802:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763732.541806:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763732.541813:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763732.541820:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763732.541829:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763732.541835:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316119584
00000100:00000001:1.0:1787763732.541842:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316120819
00000100:00000001:1.0:1787763732.541849:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316119584
00000100:00000001:1.0:1787763732.541855:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316120819
00000100:00000001:1.0:1787763732.541861:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763732.541867:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763732.541873:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763732.541881:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763732/real 1787763732]  req@ffff9751f87bfc00 x1874599832654336/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763732.541894:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763732.541896:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763732.541898:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763732.541901:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763732.541903:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763732.541905:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763732.541909:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bfc00 x1874599832654336/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763732.541916:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bfc00 x1874599832654336/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.541923:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763732.541924:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.541926:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763732.541928:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763732.541932:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bfc00 x1874599832654336/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.541940:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763732.541944:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bfc00 x1874599832654336/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.541952:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763732.541955:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.541956:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763732.541958:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.541970:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763732.541985:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763732.541988:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763732.541991:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763732.541994:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763732.541998:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bfc00 x1874599832654336/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763732.542007:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832654336:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763732.542011:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.542013:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763732.542017:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bfc00 x1874599832654336/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.542023:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763732.542025:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763732.542027:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787763732.542030:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763732.542031:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763732.542033:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763732.542035:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763732.542048:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611000.
02000000:00000001:1.0:1787763732.542051:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763732.542052:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763732.542054:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bfc00.
00000100:00000001:1.0:1787763732.542057:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763732.542059:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.542062:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.542066:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763732.542067:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763732.542069:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763732.542070:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763732.542071:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763732.542090:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc3c0 x1874599832654464/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.542098:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763732.542100:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763732.542103:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763732.542105:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763732.542107:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763732.542110:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832654464:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763732.542114:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763732.542116:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763732.542118:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763732.542120:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763732.542124:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787763732.542126:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763732.542130:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763732.542171:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763732.542176:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763732.542179:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832654464, portal 10
00000100:00000001:1.0:1787763732.542181:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763732.542182:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049846720 : -115096659704896 : ffff9751f87bc3c0)
00000100:00000040:1.0:1787763732.542188:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc3c0 x1874599832654464/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763757 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.542196:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763732.542199:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763732.542201:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763732.542204:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832654464, offset 0
00000400:00000200:1.0:1787763732.542208:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763732.542225:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763732.542230:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763732.542234:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763732.542236:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763732.542239:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763732.542242:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763732.542245:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58348:58348:256:4294967295] 192.168.203.110@tcp LPNI seq info [58348:58348:8:4294967295]
00000400:00000200:1.0:1787763732.542249:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763732.542255:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763732.542260:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763732.542264:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763732.542269:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763732.542272:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787763732.542277:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763732.542280:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787763732.542285:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763732.542287:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763732.542290:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763732.542294:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763732.542297:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763732.542299:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763732.542302:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763732.542306:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc3c0 x1874599832654464/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763757 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.542314:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763732.542317:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc3c0 x1874599832654464/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763757 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.542322:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763732.542323:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763732.542326:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763732.542328:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763732.542330:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763732.542331:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316119573
00000100:00000001:1.0:1787763732.542334:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316120819
00000100:00000001:1.0:1787763732.542335:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316119573
00000100:00000001:1.0:1787763732.542347:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316120819
00000100:00000001:1.0:1787763732.542349:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763732.542350:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763732.542351:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763732.542354:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763732/real 1787763732]  req@ffff9751f87bc3c0 x1874599832654464/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763732.542358:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763732.542359:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763732.542360:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763732.542362:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763732.542363:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763732.542364:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763732.542366:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc3c0 x1874599832654464/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763732.542372:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc3c0 x1874599832654464/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.542375:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763732.542376:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.542377:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763732.542378:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763732.542381:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc3c0 x1874599832654464/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.542385:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763732.542387:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc3c0 x1874599832654464/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.542392:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763732.542394:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.542396:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763732.542397:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.542398:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763732.542400:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763732.542402:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763732.542404:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763732.542405:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763732.542408:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc3c0 x1874599832654464/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763732.542412:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832654464:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763732.542415:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763732.542417:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc780 x1874599832654592/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.542421:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763732.542422:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763732.542423:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763732.542424:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763732.542426:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763732.542427:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832654592:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763732.542430:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763732.542432:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763732.542433:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763732.542434:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763732.542437:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763732.542438:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763732.542440:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763732.542442:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763732.542445:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763732.542447:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832654592, portal 4
00000100:00000001:1.0:1787763732.542448:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763732.542450:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049847680 : -115096659703936 : ffff9751f87bc780)
00000100:00000040:1.0:1787763732.542453:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc780 x1874599832654592/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763757 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.542468:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763732.542470:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763732.542471:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763732.542473:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832654592, offset 0
00000400:00000200:1.0:1787763732.542475:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763732.542478:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763732.542480:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763732.542482:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763732.542483:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763732.542485:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763732.542486:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763732.542489:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58349:58349:256:4294967295] 192.168.203.110@tcp LPNI seq info [58349:58349:8:4294967295]
00000400:00000200:1.0:1787763732.542491:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763732.542496:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763732.542499:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763732.542501:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763732.542503:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763732.542505:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787763732.542508:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763732.542510:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787763732.542512:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763732.542514:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763732.542516:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763732.542518:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763732.542520:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763732.542521:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763732.542523:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763732.542525:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc780 x1874599832654592/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763757 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.542530:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763732.542532:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc780 x1874599832654592/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763757 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.542549:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763732.542550:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763732.542552:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763732.542554:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763732.542556:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763732.542557:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316119590
00000100:00000001:1.0:1787763732.542559:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316120819
00000100:00000001:1.0:1787763732.542560:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316119590
00000100:00000001:1.0:1787763732.542562:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316120819
00000100:00000001:1.0:1787763732.542564:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763732.542565:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763732.542566:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763732.542568:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763732/real 1787763732]  req@ffff9751f87bc780 x1874599832654592/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763732.542572:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763732.542573:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763732.542574:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763732.542575:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763732.542576:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763732.542577:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763732.542579:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc780 x1874599832654592/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763732.542584:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc780 x1874599832654592/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.542589:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763732.542590:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.542591:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763732.542592:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763732.542595:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc780 x1874599832654592/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.542600:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763732.542602:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc780 x1874599832654592/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.542607:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763732.542608:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.542609:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763732.542610:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.542611:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763732.542612:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763732.542613:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763732.542615:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763732.542616:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763732.542619:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc780 x1874599832654592/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763732.542623:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832654592:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763732.542626:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.542627:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763732.542629:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc3c0 x1874599832654464/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.542635:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763732.542637:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763732.542639:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787763732.542641:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763732.542643:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763732.542644:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763732.542646:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763732.542649:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610000.
02000000:00000001:1.0:1787763732.542652:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763732.542653:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763732.542655:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc3c0.
00000100:00000001:1.0:1787763732.542657:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763732.542659:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.542661:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763732.542665:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc780 x1874599832654592/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763757 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763732.542672:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763732.542673:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763732.542675:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763732.542677:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763732.542679:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763732.542680:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763732.542682:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763732.542684:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613400.
02000000:00000001:1.0:1787763732.542687:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763732.542688:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763732.542690:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc780.
00000100:00000001:1.0:1787763732.542691:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763732.542692:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.542693:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763732.542696:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763732.542698:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763732.542699:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763732.542701:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763733.180224:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763733.180240:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763733.180246:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763733.180256:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763733.180266:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763733.180271:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763733.180276:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763733.180281:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763733.180285:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763733.564261:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763733.564272:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763733.564275:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763733.564284:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763733.564290:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763733.564301:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763733.564305:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763733.564309:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763733.564315:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763734.204225:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763734.204234:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763734.204238:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763734.204247:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763734.204254:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763734.204261:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763734.204267:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763734.204273:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763734.204277:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763734.588205:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763734.588215:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763734.588218:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763734.588224:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763734.588229:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763734.588233:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763734.588237:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763734.588241:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763734.588245:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763735.228210:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763735.228220:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763735.228224:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763735.228228:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763735.228235:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763735.228240:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763735.228244:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763735.228246:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763735.228248:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763735.228295:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763735.228302:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763735.228306:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763735.612264:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763735.612274:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763735.612277:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763735.612283:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763735.612287:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763735.612299:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763735.612302:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763735.612306:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763735.612309:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763736.252235:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763736.252245:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763736.252249:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763736.252254:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763736.252260:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763736.252266:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763736.252271:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763736.252274:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763736.252281:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763736.636208:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763736.636217:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763736.636219:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763736.636225:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763736.636230:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763736.636234:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763736.636236:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763736.636239:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763736.636242:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763737.276194:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763737.276204:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763737.276209:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763737.276214:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763737.276228:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763737.276232:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.276235:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763737.276238:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.276241:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763737.660247:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763737.660257:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763737.660262:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763737.660270:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763737.660277:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763737.660283:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763737.660290:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763737.660295:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763737.660302:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763737.660485:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763737.660492:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763737.660497:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763737.660509:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bcf00 x1874599832654720/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.660528:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763737.660533:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763737.660537:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763737.660542:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763737.660546:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763737.660553:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832654720:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763737.660558:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763737.660564:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763737.660569:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763737.660573:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763737.660579:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763737.660584:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763737.660592:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763737.660599:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763737.660612:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763737.660623:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832654720, portal 25
00000100:00000001:1.0:1787763737.660627:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763737.660629:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049849600 : -115096659702016 : ffff9751f87bcf00)
00000100:00000040:1.0:1787763737.660637:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bcf00 x1874599832654720/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763762 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.660653:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763737.660660:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763737.660667:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763737.660673:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832654720, offset 0
00000400:00000200:1.0:1787763737.660687:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763737.660702:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763737.660715:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763737.660724:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763737.660741:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763737.660749:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763737.660757:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763737.660767:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58350:58350:256:4294967295] 192.168.203.110@tcp LPNI seq info [58350:58350:8:4294967295]
00000400:00000200:1.0:1787763737.660775:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763737.660793:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763737.660804:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763737.660812:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763737.660823:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763737.660830:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787763737.660839:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763737.660844:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787763737.660851:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763737.660855:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763737.660864:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763737.660874:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763737.660883:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763737.660887:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763737.660893:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763737.660901:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bcf00 x1874599832654720/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763762 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.660919:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763737.660927:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bcf00 x1874599832654720/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763762 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.660934:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763737.660937:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763737.660946:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763737.660957:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763737.660961:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316125939
00000100:00000010:1.0:1787763737.660970:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4360.
00000100:00000001:1.0:1787763737.660983:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787763737.660994:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316125939
00000100:00000010:1.0:1787763737.660999:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4600.
00000100:00000001:1.0:1787763737.661007:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763737.661014:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763737.661021:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763737.661024:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763737.661032:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763737/real 1787763737]  req@ffff9751f87bcf00 x1874599832654720/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763737.661061:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763737.661065:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763737.661068:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763737.661072:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763737.661076:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763737.661080:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763737.661087:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bcf00 x1874599832654720/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763737.661100:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bcf00 x1874599832654720/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661110:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763737.661113:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.661118:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763737.661121:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763737.661131:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bcf00 x1874599832654720/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661158:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763737.661165:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bcf00 x1874599832654720/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661172:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763737.661175:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.661176:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763737.661177:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.661180:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763737.661181:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763737.661183:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763737.661186:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763737.661189:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763737.661192:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bcf00 x1874599832654720/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763737.661197:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832654720:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763737.661201:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.661202:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763737.661204:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bcf00 x1874599832654720/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661209:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763737.661212:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763737.661214:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763737.661216:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763737.661218:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763737.661219:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763737.661221:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763737.661223:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613000.
02000000:00000001:1.0:1787763737.661227:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763737.661228:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763737.661230:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bcf00.
00000100:00000001:1.0:1787763737.661233:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763737.661234:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.661235:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.661242:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763737.661244:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763737.661245:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763737.661247:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763737.661248:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763737.661250:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc780 x1874599832654848/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661255:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763737.661257:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763737.661258:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763737.661260:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763737.661262:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763737.661264:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832654848:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763737.661266:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763737.661267:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763737.661269:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763737.661270:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763737.661272:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763737.661274:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763737.661276:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763737.661278:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763737.661281:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763737.661283:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832654848, portal 10
00000100:00000001:1.0:1787763737.661285:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763737.661286:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049847680 : -115096659703936 : ffff9751f87bc780)
00000100:00000040:1.0:1787763737.661292:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc780 x1874599832654848/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763762 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661296:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763737.661298:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763737.661299:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763737.661301:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832654848, offset 0
00000400:00000200:1.0:1787763737.661303:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763737.661306:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763737.661308:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763737.661310:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763737.661312:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763737.661314:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763737.661315:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763737.661318:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58351:58351:256:4294967295] 192.168.203.110@tcp LPNI seq info [58351:58351:8:4294967295]
00000400:00000200:1.0:1787763737.661320:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763737.661324:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763737.661327:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763737.661331:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763737.661333:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763737.661335:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787763737.661339:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763737.661341:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787763737.661343:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763737.661345:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763737.661347:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763737.661349:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763737.661352:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763737.661353:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763737.661355:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763737.661357:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc780 x1874599832654848/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763762 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661363:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763737.661365:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc780 x1874599832654848/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763762 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661370:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763737.661371:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763737.661372:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763737.661376:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763737.661378:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763737.661379:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316124704
00000100:00000001:1.0:1787763737.661381:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316125939
00000100:00000001:1.0:1787763737.661383:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316124704
00000100:00000001:1.0:1787763737.661384:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316125939
00000100:00000001:1.0:1787763737.661386:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763737.661388:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763737.661389:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763737.661391:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763737/real 1787763737]  req@ffff9751f87bc780 x1874599832654848/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763737.661401:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763737.661402:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763737.661403:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763737.661405:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763737.661406:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763737.661407:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763737.661409:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc780 x1874599832654848/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763737.661414:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc780 x1874599832654848/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661418:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763737.661419:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.661420:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763737.661421:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763737.661424:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc780 x1874599832654848/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661428:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763737.661430:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc780 x1874599832654848/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661434:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763737.661436:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.661437:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763737.661438:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.661439:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763737.661450:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763737.661451:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763737.661453:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763737.661455:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763737.661459:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc780 x1874599832654848/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763737.661464:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832654848:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763737.661467:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763737.661469:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc3c0 x1874599832654976/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661473:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763737.661474:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763737.661475:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763737.661476:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763737.661477:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763737.661479:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832654976:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763737.661481:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763737.661482:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763737.661483:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763737.661484:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763737.661486:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787763737.661487:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763737.661489:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763737.661490:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763737.661492:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763737.661494:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832654976, portal 10
00000100:00000001:1.0:1787763737.661495:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763737.661496:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049846720 : -115096659704896 : ffff9751f87bc3c0)
00000100:00000040:1.0:1787763737.661501:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc3c0 x1874599832654976/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763762 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661506:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763737.661507:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763737.661508:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763737.661510:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832654976, offset 0
00000400:00000200:1.0:1787763737.661513:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763737.661515:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763737.661518:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763737.661520:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763737.661522:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763737.661523:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763737.661525:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763737.661527:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58352:58352:256:4294967295] 192.168.203.110@tcp LPNI seq info [58352:58352:8:4294967295]
00000400:00000200:1.0:1787763737.661530:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763737.661534:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763737.661537:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763737.661539:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763737.661541:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763737.661543:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787763737.661546:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763737.661547:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787763737.661550:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763737.661552:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763737.661554:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763737.661556:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763737.661558:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763737.661559:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763737.661561:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763737.661563:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc3c0 x1874599832654976/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763762 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661568:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763737.661570:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc3c0 x1874599832654976/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763762 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661574:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763737.661575:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763737.661577:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763737.661579:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763737.661580:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763737.661582:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316124692
00000100:00000001:1.0:1787763737.661584:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316125939
00000100:00000001:1.0:1787763737.661586:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316124692
00000100:00000001:1.0:1787763737.661588:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316125939
00000100:00000001:1.0:1787763737.661589:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763737.661590:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763737.661591:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763737.661594:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763737/real 1787763737]  req@ffff9751f87bc3c0 x1874599832654976/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763737.661598:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763737.661599:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763737.661600:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763737.661601:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763737.661602:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763737.661604:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763737.661606:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc3c0 x1874599832654976/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763737.661612:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc3c0 x1874599832654976/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661615:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763737.661616:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.661617:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763737.661618:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763737.661620:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc3c0 x1874599832654976/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661624:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763737.661626:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc3c0 x1874599832654976/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661630:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763737.661633:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.661634:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763737.661635:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.661636:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763737.661637:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763737.661639:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763737.661641:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763737.661642:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763737.661644:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc3c0 x1874599832654976/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763737.661649:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832654976:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763737.661651:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763737.661653:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bfc00 x1874599832655104/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661657:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763737.661658:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763737.661659:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763737.661660:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763737.661661:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763737.661663:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832655104:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763737.661666:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763737.661667:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763737.661668:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763737.661669:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763737.661671:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763737.661672:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763737.661674:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763737.661676:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763737.661678:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763737.661680:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832655104, portal 4
00000100:00000001:1.0:1787763737.661682:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763737.661683:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049861120 : -115096659690496 : ffff9751f87bfc00)
00000100:00000040:1.0:1787763737.661687:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bfc00 x1874599832655104/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763762 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661691:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763737.661693:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763737.661694:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763737.661695:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832655104, offset 0
00000400:00000200:1.0:1787763737.661697:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763737.661699:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763737.661701:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763737.661703:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763737.661705:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763737.661707:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763737.661708:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763737.661710:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58353:58353:256:4294967295] 192.168.203.110@tcp LPNI seq info [58353:58353:8:4294967295]
00000400:00000200:1.0:1787763737.661713:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763737.661717:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763737.661720:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763737.661722:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763737.661724:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763737.661725:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787763737.661728:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763737.661730:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787763737.661733:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763737.661734:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763737.661736:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763737.661738:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763737.661740:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763737.661741:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763737.661743:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763737.661745:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bfc00 x1874599832655104/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763762 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661755:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763737.661757:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bfc00 x1874599832655104/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763762 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661761:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763737.661762:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763737.661764:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763737.661766:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763737.661768:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763737.661769:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316124709
00000100:00000001:1.0:1787763737.661770:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316125939
00000100:00000001:1.0:1787763737.661771:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316124709
00000100:00000001:1.0:1787763737.661773:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316125939
00000100:00000001:1.0:1787763737.661775:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763737.661777:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763737.661778:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763737.661780:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763737/real 1787763737]  req@ffff9751f87bfc00 x1874599832655104/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763737.661784:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763737.661785:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763737.661786:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763737.661787:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763737.661788:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763737.661789:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763737.661791:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bfc00 x1874599832655104/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763737.661796:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bfc00 x1874599832655104/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661800:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763737.661802:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.661803:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763737.661804:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763737.661806:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bfc00 x1874599832655104/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661811:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763737.661813:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bfc00 x1874599832655104/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661817:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763737.661818:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.661819:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763737.661820:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.661821:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763737.661822:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763737.661823:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763737.661825:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763737.661826:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763737.661829:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bfc00 x1874599832655104/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763737.661833:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832655104:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763737.661835:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.661837:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763737.661838:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc780 x1874599832654848/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661843:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763737.661844:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763737.661845:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763737.661847:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763737.661848:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763737.661850:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763737.661851:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763737.661852:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612800.
02000000:00000001:1.0:1787763737.661854:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763737.661855:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763737.661856:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc780.
00000100:00000001:1.0:1787763737.661858:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763737.661859:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.661860:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763737.661862:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc3c0 x1874599832654976/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661867:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763737.661868:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763737.661868:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787763737.661870:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763737.661871:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763737.661871:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763737.661872:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763737.661873:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612c00.
02000000:00000001:1.0:1787763737.661874:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763737.661875:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763737.661876:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc3c0.
00000100:00000001:1.0:1787763737.661877:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763737.661878:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.661879:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763737.661880:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bfc00 x1874599832655104/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763762 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763737.661884:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763737.661885:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763737.661886:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763737.661887:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763737.661887:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763737.661888:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763737.661889:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763737.661890:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612400.
02000000:00000001:1.0:1787763737.661891:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763737.661892:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763737.661893:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bfc00.
00000100:00000001:1.0:1787763737.661895:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763737.661896:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.661897:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763737.661900:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763737.661901:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763737.661902:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763737.661903:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763738.300271:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763738.300282:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763738.300285:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763738.300291:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763738.300296:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763738.300300:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763738.300304:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763738.300307:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763738.300327:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763738.300350:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763738.300356:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763738.300360:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763738.684215:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763738.684224:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763738.684225:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763738.684231:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763738.684235:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763738.684239:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763738.684243:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763738.684246:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763738.684249:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763739.324290:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763739.324318:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763739.324323:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763739.324331:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763739.324339:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763739.324345:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763739.324354:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763739.324359:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763739.324362:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763739.708258:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763739.708266:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763739.708268:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763739.708274:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763739.708278:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763739.708282:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763739.708285:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763739.708288:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763739.708291:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763740.348241:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763740.348253:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763740.348257:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763740.348264:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763740.348270:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763740.348276:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763740.348279:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763740.348284:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763740.348288:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763740.732200:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763740.732208:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763740.732210:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763740.732215:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763740.732219:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763740.732222:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763740.732225:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763740.732227:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763740.732231:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763741.372256:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763741.372267:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763741.372274:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763741.372286:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763741.372304:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763741.372312:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763741.372318:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763741.372322:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763741.372326:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763741.372388:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763741.372395:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763741.372400:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763741.756282:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763741.756293:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763741.756295:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763741.756303:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763741.756307:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763741.756311:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763741.756315:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763741.756319:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763741.756322:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763742.396294:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763742.396307:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763742.396315:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763742.396323:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763742.396338:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763742.396348:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.396354:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763742.396357:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.396362:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763742.396424:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763742.396435:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763742.396442:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763742.780620:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763742.780630:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763742.780638:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763742.780653:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832655232/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.780687:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763742.780692:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763742.780703:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763742.780709:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763742.780715:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763742.780726:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832655232:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763742.780752:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763742.780760:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763742.780765:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763742.780772:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763742.780783:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763742.780793:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763742.780810:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763742.780819:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763742.780831:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763742.780844:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832655232, portal 25
00000100:00000001:1.0:1787763742.780853:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763742.780857:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:1.0:1787763742.780872:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832655232/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763767 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.780901:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763742.780912:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763742.780916:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763742.780925:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832655232, offset 0
00000400:00000200:1.0:1787763742.780939:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763742.780954:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763742.780972:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763742.780986:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763742.780999:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763742.781031:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763742.781041:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763742.781054:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58354:58354:256:4294967295] 192.168.203.110@tcp LPNI seq info [58354:58354:8:4294967295]
00000400:00000200:1.0:1787763742.781068:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763742.781093:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763742.781113:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763742.781126:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763742.781181:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763742.781189:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787763742.781205:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763742.781216:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787763742.781233:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763742.781242:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763742.781255:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763742.781270:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763742.781280:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763742.781288:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763742.781298:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763742.781311:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832655232/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763767 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.781341:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763742.781350:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832655232/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763767 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.781367:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763742.781374:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763742.781384:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763742.781398:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763742.781406:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316131060
00000100:00000010:1.0:1787763742.781420:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4900.
00000100:00000001:1.0:1787763742.781434:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787763742.781448:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316131060
00000100:00000010:1.0:1787763742.781458:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4780.
00000100:00000001:1.0:1787763742.781466:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763742.781473:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763742.781479:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763742.781490:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763742.781506:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763742/real 1787763742]  req@ffff9751f86583c0 x1874599832655232/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763742.781530:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763742.781537:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763742.781546:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763742.781558:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763742.781569:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763742.781582:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763742.781598:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832655232/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763742.781640:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832655232/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.781667:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763742.781676:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.781685:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763742.781689:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763742.781700:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832655232/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.781725:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763742.781736:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832655232/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.781760:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763742.781769:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.781776:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763742.781780:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.781785:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763742.781791:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763742.781796:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763742.781807:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763742.781815:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763742.781827:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832655232/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763742.781870:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832655232:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763742.781886:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.781891:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763742.781901:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832655232/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.781930:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763742.781937:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763742.781941:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763742.781950:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763742.781952:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763742.781955:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763742.781958:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763742.781962:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:1.0:1787763742.781967:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763742.781969:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763742.781972:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:1.0:1787763742.781975:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763742.781977:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.781979:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.781987:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763742.781988:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763742.781991:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763742.781993:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763742.781995:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763742.781999:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832655360/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.782030:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763742.782032:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763742.782035:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763742.782037:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763742.782040:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763742.782044:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832655360:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763742.782048:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763742.782051:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763742.782053:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763742.782056:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763742.782060:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763742.782062:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763742.782067:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763742.782070:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763742.782075:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763742.782079:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832655360, portal 10
00000100:00000001:1.0:1787763742.782081:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763742.782083:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787763742.782092:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832655360/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763767 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.782106:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763742.782110:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763742.782114:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763742.782119:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832655360, offset 0
00000400:00000200:1.0:1787763742.782127:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763742.782167:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763742.782176:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763742.782182:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763742.782187:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763742.782192:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763742.782195:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763742.782199:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58355:58355:256:4294967295] 192.168.203.110@tcp LPNI seq info [58355:58355:8:4294967295]
00000400:00000200:1.0:1787763742.782205:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763742.782214:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763742.782220:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763742.782225:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763742.782231:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763742.782234:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787763742.782240:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763742.782243:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787763742.782248:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763742.782250:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763742.782254:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763742.782259:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763742.782263:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763742.782265:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763742.782269:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763742.782273:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832655360/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763767 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.782284:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763742.782287:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832655360/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763767 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.782303:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763742.782305:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763742.782310:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763742.782314:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763742.782318:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763742.782321:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316129825
00000100:00000001:1.0:1787763742.782324:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316131060
00000100:00000001:1.0:1787763742.782327:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316129825
00000100:00000001:1.0:1787763742.782330:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316131060
00000100:00000001:1.0:1787763742.782332:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763742.782335:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763742.782337:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763742.782342:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763742/real 1787763742]  req@ffff9751f8658000 x1874599832655360/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763742.782351:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763742.782353:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763742.782355:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763742.782357:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763742.782359:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763742.782362:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763742.782366:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832655360/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763742.782375:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832655360/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.782382:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763742.782383:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.782385:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763742.782387:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763742.782391:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832655360/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.782400:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763742.782404:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832655360/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.782412:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763742.782415:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.782417:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763742.782418:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.782420:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763742.782434:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763742.782437:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763742.782440:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763742.782443:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763742.782447:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832655360/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763742.782456:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832655360:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763742.782461:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763742.782465:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832655488/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.782472:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763742.782474:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763742.782476:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763742.782478:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763742.782481:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763742.782484:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832655488:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763742.782487:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763742.782490:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763742.782493:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763742.782495:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763742.782499:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787763742.782501:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763742.782505:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763742.782508:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763742.782511:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763742.782515:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832655488, portal 10
00000100:00000001:1.0:1787763742.782517:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763742.782518:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787763742.782525:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832655488/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763767 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.782537:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763742.782541:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763742.782545:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763742.782550:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832655488, offset 0
00000400:00000200:1.0:1787763742.782557:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763742.782564:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763742.782570:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763742.782576:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763742.782581:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763742.782585:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763742.782590:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763742.782596:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58356:58356:256:4294967295] 192.168.203.110@tcp LPNI seq info [58356:58356:8:4294967295]
00000400:00000200:1.0:1787763742.782604:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763742.782616:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763742.782625:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763742.782629:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763742.782634:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763742.782637:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787763742.782642:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763742.782646:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787763742.782650:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763742.782653:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763742.782657:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763742.782661:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763742.782665:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763742.782667:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763742.782670:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763742.782675:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832655488/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763767 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.782685:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763742.782706:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832655488/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763767 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.782714:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763742.782716:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763742.782720:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763742.782724:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763742.782729:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763742.782731:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316129813
00000100:00000001:1.0:1787763742.782735:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316131060
00000100:00000001:1.0:1787763742.782737:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316129813
00000100:00000001:1.0:1787763742.782740:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316131060
00000100:00000001:1.0:1787763742.782743:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763742.782745:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763742.782748:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763742.782752:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763742/real 1787763742]  req@ffff9751f8659a40 x1874599832655488/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763742.782761:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763742.782763:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763742.782765:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763742.782768:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763742.782787:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763742.782790:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763742.782793:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832655488/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763742.782803:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832655488/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.782810:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763742.782812:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.782814:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763742.782815:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763742.782819:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832655488/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.782826:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763742.782831:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832655488/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.782838:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763742.782841:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.782843:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763742.782845:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.782847:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763742.782849:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763742.782852:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763742.782855:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763742.782858:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763742.782862:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832655488/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763742.782871:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832655488:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763742.782875:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763742.782879:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832655616/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.782887:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763742.782889:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763742.782891:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763742.782894:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763742.782896:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763742.782899:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832655616:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763742.782902:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763742.782905:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763742.782907:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763742.782909:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763742.782913:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763742.782915:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763742.782919:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763742.782922:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763742.782925:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763742.782929:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832655616, portal 4
00000100:00000001:1.0:1787763742.782931:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763742.782932:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:1.0:1787763742.782939:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832655616/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763767 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.782946:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763742.782949:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763742.782951:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763742.782954:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832655616, offset 0
00000400:00000200:1.0:1787763742.782958:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763742.782962:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763742.782966:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763742.782969:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763742.782972:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763742.782975:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763742.782978:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763742.782982:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58357:58357:256:4294967295] 192.168.203.110@tcp LPNI seq info [58357:58357:8:4294967295]
00000400:00000200:1.0:1787763742.782987:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763742.782995:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763742.783000:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763742.783029:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763742.783035:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763742.783038:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787763742.783043:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763742.783046:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787763742.783051:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763742.783053:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763742.783057:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763742.783061:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763742.783065:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763742.783067:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763742.783070:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763742.783074:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832655616/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763767 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.783083:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763742.783087:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832655616/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763767 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.783097:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763742.783100:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763742.783106:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763742.783111:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763742.783117:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763742.783120:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316129830
00000100:00000001:1.0:1787763742.783124:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316131060
00000100:00000001:1.0:1787763742.783128:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316129830
00000100:00000001:1.0:1787763742.783161:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316131060
00000100:00000001:1.0:1787763742.783167:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763742.783172:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763742.783176:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763742.783185:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763742/real 1787763742]  req@ffff9751f865ad00 x1874599832655616/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763742.783197:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763742.783198:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763742.783200:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763742.783204:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763742.783206:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763742.783208:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763742.783212:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832655616/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763742.783222:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832655616/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.783230:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763742.783232:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.783234:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763742.783235:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763742.783239:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832655616/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.783246:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763742.783251:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832655616/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.783259:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763742.783262:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.783264:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763742.783265:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.783268:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763742.783269:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763742.783272:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763742.783277:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763742.783279:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763742.783284:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832655616/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763742.783292:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832655616:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763742.783296:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.783299:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763742.783302:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832655360/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.783310:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763742.783312:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763742.783314:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763742.783317:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763742.783319:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763742.783321:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763742.783323:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763742.783325:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:1.0:1787763742.783329:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763742.783330:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763742.783332:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:1.0:1787763742.783335:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763742.783336:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.783338:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763742.783342:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832655488/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.783349:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763742.783350:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763742.783352:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787763742.783354:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763742.783355:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763742.783357:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763742.783359:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763742.783361:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:1.0:1787763742.783364:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763742.783365:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763742.783367:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787763742.783369:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763742.783371:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.783372:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763742.783376:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832655616/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763767 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763742.783382:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763742.783384:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763742.783385:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763742.783387:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763742.783388:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763742.783390:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763742.783392:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763742.783393:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:1.0:1787763742.783395:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763742.783396:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763742.783398:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:1.0:1787763742.783400:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763742.783402:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.783404:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763742.783408:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763742.783410:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763742.783412:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763742.783414:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763743.420246:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763743.420255:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763743.420259:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763743.420266:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763743.420273:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763743.420278:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763743.420283:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763743.420288:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763743.420296:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763744.444279:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763744.444290:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763744.444296:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763744.444301:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763744.444306:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763744.444311:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763744.444314:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763744.444317:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763744.444321:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763745.468268:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763745.468279:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763745.468287:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763745.468299:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763745.468307:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763745.468316:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763745.468324:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763745.468330:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763745.468337:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763746.492226:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763746.492236:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763746.492239:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763746.492248:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763746.492255:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763746.492262:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763746.492267:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763746.492271:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763746.492275:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763747.516326:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763747.516336:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763747.516339:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763747.516346:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763747.516353:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763747.516359:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763747.516363:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763747.516367:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763747.516372:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763747.900303:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763747.900315:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763747.900319:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763747.900325:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763747.900330:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763747.900340:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21462
00000400:00000200:0.0:1787763747.900352:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763747.900369:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763747.900373:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763747.900376:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763747.900379:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763747.900382:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763747.900384:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763747.900389:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763747.900391:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763747.900394:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763747.900398:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763747.900400:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763747.900403:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763747.900406:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763747.900407:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763747.900409:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763747.900411:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763747.900414:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763747.900418:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763747.900420:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763747.900423:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763747.900428:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763747.900431:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763747.900433:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763747.900436:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763747.900449:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763747.900453:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763747.900457:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763747.900460:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763747.900463:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763747.900467:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763747.900472:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763747.900483:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763747.900491:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763747.900502:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832655744/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763747.900519:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763747.900525:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763747.900531:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832655744/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763747.900558:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763747.900561:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763747.900565:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763747.900566:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763747.900568:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763747.900571:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763747.900574:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21462
00000400:00000200:0.0:1787763747.900577:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763747.900580:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763747.900582:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763747.900585:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763747.900587:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763747.900589:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763747.900591:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763747.900593:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763747.900594:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763747.900595:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763747.900597:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763747.900599:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763747.900601:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763747.900604:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763747.900606:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763747.900608:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763747.900610:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763747.900612:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763747.900615:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763747.900617:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763747.900618:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763747.900625:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763747.900627:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763747.900630:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763747.900637:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763747.900640:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763747.900641:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763747.900647:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000100:00000001:1.0:1787763747.900648:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:0.0:1787763747.900651:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763747.900660:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1787763747.900661:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
02000000:00000001:0.0:1787763747.900663:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763747.900668:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1787763747.900671:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763747.900676:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763747.900679:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763747.900685:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000040:1.0:1787763747.900688:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832655744/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763747.900696:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763747.900713:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763747.900716:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763747.900725:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832655872/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1787763747.900726:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763747.900733:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763747.900742:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763747.900759:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832655744:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763747.900764:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763747.900767:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1787763747.900784:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763747.900785:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763747.900793:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763747.900803:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832655872/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787763747.900804:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1787763747.900830:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763747.900833:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
00000100:00080000:0.0:1787763747.900838:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:1.0:1787763747.900839:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763747.900870:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:0.0:1787763747.900874:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000010:1.0:1787763747.900879:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000100:00000001:0.0:1787763747.900881:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763747.900888:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:1.0:1787763747.900890:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000001:0.0:1787763747.900905:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:1.0:1787763747.900908:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832655744, portal 25
00000100:00000001:1.0:1787763747.900916:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00080000:0.0:1787763747.900917:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21462
00000100:00000001:1.0:1787763747.900920:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000400:00000200:0.0:1787763747.900944:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:1.0:1787763747.900944:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832655744/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763772 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787763747.900987:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763747.900991:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763747.900994:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763747.901004:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763747.901011:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763747.901018:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763747.901021:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763747.901032:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763747.901034:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:1.0:1787763747.901039:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000040:0.0:1787763747.901040:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763747.901047:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763747.901056:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787763747.901057:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787763747.901063:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763747.901066:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:1.0:1787763747.901067:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000040:0.0:1787763747.901069:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763747.901076:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763747.901079:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00000200:1.0:1787763747.901082:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832655744, offset 0
00000100:00080000:0.0:1787763747.901083:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763747.901096:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763747.901100:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763747.901105:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1787763747.901106:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
02000000:00000001:0.0:1787763747.901111:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763747.901114:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763747.901118:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:1.0:1787763747.901127:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000008:00000001:0.0:1787763747.901128:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763747.901141:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763747.901191:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1787763747.901194:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763747.901202:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763747.901208:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000400:00000200:1.0:1787763747.901209:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
02000000:00000001:0.0:1787763747.901225:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763747.901227:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763747.901232:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1787763747.901233:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763747.901237:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763747.901242:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763747.901243:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
02000000:00000010:0.0:1787763747.901261:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000400:00000200:1.0:1787763747.901263:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58358:58358:256:4294967295] 192.168.203.110@tcp LPNI seq info [58358:58358:8:4294967295]
00000100:00000001:0.0:1787763747.901273:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763747.901279:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00100000:0.0:1787763747.901287:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832656000/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787763747.901308:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1787763747.901330:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763747.901341:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787763747.901341:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787763747.901356:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832656000/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:1.0:1787763747.901358:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000100:00000001:0.0:1787763747.901377:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787763747.901380:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763747.901382:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:1.0:1787763747.901387:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000100:00000040:0.0:1787763747.901389:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763747.901404:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:1.1:1787763747.901417:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00080000:0.0:1787763747.901421:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763747.901425:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:1.1:1787763747.901426:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000100:00080000:0.0:1787763747.901431:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763747.901436:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763747.901447:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21462
00000800:00000100:1.0:1787763747.901454:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763747.901462:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000010:1.0:1787763747.901466:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:0.0:1787763747.901470:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763747.901476:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763747.901481:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:1.0:1787763747.901483:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1787763747.901485:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763747.901491:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763747.901494:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763747.901501:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:1.0:1787763747.901503:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000020:00000040:0.0:1787763747.901504:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763747.901510:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763747.901518:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763747.901523:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000200:1.0:1787763747.901525:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000020:00000001:0.0:1787763747.901527:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000010:1.0:1787763747.901537:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000040:0.0:1787763747.901538:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763747.901545:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763747.901551:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763747.901552:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1787763747.901554:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763747.901558:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763747.901565:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000200:1.0:1787763747.901566:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e1c0 x1874599832655744/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763772 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763747.901577:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763747.901584:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763747.901589:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763747.901597:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763747.901609:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763747.901612:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763747.901616:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763747.901618:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763747.901619:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000010:0.0:1787763747.901623:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763747.901627:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763747.901633:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:1.0:1787763747.901636:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832655744/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763772 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763747.901637:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763747.901646:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763747.901648:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763747.901656:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763747.901673:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:1.0:1787763747.901676:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763747.901680:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763747.901683:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763747.901690:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763747.901699:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832656128/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787763747.901699:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763747.901706:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316136180
00000100:00000010:1.0:1787763747.901718:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4d20.
00000100:00000001:0.0:1787763747.901730:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763747.901730:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763747.901740:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763747.901748:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316136180
00000100:00000040:0.0:1787763747.901750:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832656128/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:1.0:1787763747.901755:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c48a0.
00000100:00000001:1.0:1787763747.901768:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763747.901777:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763747.901797:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763747.901798:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763747.901810:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000040:0.0:1787763747.901814:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21472)
00000100:00100000:1.0:1787763747.901821:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763747/real 1787763747]  req@ffff9751f876e1c0 x1874599832655744/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763747.901831:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763747.901834:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763747.901836:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763747.901840:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763747.901842:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763747.901854:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763747.901857:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832655744/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763747.901868:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832655744/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.901876:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763747.901878:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763747.901880:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763747.901882:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763747.901886:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832655744/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.901892:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763747.901897:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832655744/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.901904:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763747.901906:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763747.901908:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763747.901909:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763747.901912:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763747.901914:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763747.901918:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763747.901922:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763747.901925:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763747.901929:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832655744/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763747.901936:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832655744:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763747.901940:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763747.901943:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763747.901946:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832655744/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.901951:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763747.901954:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763747.901957:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763747.901960:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763747.901964:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763747.901984:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763747.901989:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763747.901992:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014800.
02000000:00000001:1.0:1787763747.901998:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763747.902000:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763747.902003:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:1.0:1787763747.902009:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763747.902011:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763747.902014:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763747.902024:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763747.902026:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763747.902029:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763747.902032:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763747.902034:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763747.902041:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832655872/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.902059:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763747.902061:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763747.902064:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763747.902066:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763747.902068:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763747.902071:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832655872:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763747.902074:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763747.902077:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763747.902079:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763747.902081:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763747.902084:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763747.902086:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763747.902090:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763747.902093:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763747.902097:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763747.902100:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832655872, portal 10
00000100:00000001:1.0:1787763747.902102:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763747.902104:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
00000100:00000040:1.0:1787763747.902110:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832655872/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763772 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.902117:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763747.902119:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763747.902121:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763747.902124:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832655872, offset 0
00000400:00000200:1.0:1787763747.902128:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763747.902160:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763747.902167:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763747.902172:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763747.902175:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763747.902179:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763747.902183:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763747.902189:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58359:58359:256:4294967295] 192.168.203.110@tcp LPNI seq info [58359:58359:8:4294967295]
00000400:00000200:1.0:1787763747.902196:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763747.902207:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763747.902215:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763747.902221:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763747.902236:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763747.902240:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787763747.902246:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763747.902249:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787763747.902253:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763747.902255:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763747.902258:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763747.902262:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763747.902265:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763747.902267:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763747.902270:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763747.902274:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876ed00 x1874599832655872/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763772 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.902282:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763747.902285:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832655872/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763772 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.902292:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763747.902293:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763747.902297:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763747.902301:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763747.902304:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763747.902306:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316134945
00000100:00000001:1.0:1787763747.902309:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316136180
00000100:00000001:1.0:1787763747.902312:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316134945
00000100:00000001:1.0:1787763747.902314:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316136180
00000100:00000001:1.0:1787763747.902316:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763747.902318:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763747.902320:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763747.902324:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763747/real 1787763747]  req@ffff9751f876ed00 x1874599832655872/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763747.902332:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763747.902333:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763747.902335:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763747.902337:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763747.902339:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763747.902340:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763747.902343:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832655872/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763747.902352:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832655872/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.902357:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763747.902358:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763747.902360:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763747.902361:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763747.902365:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832655872/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.902371:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763747.902374:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832655872/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.902380:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763747.902382:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763747.902384:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763747.902385:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763747.902387:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763747.902399:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763747.902402:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763747.902406:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763747.902408:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763747.902412:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832655872/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763747.902419:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832655872:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763747.902422:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763747.902426:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832656000/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.902432:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763747.902433:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763747.902435:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763747.902437:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763747.902439:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763747.902442:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832656000:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763747.902445:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763747.902447:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763747.902449:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763747.902451:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763747.902454:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787763747.902456:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763747.902459:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763747.902461:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763747.902463:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763747.902466:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832656000, portal 10
00000100:00000001:1.0:1787763747.902468:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763747.902470:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00000040:1.0:1787763747.902476:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832656000/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763772 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.902482:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763747.902485:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763747.902487:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763747.902489:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832656000, offset 0
00000400:00000200:1.0:1787763747.902493:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763747.902497:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763747.902500:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763747.902503:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763747.902506:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763747.902508:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763747.902510:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763747.902513:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58360:58360:256:4294967295] 192.168.203.110@tcp LPNI seq info [58360:58360:8:4294967295]
00000400:00000200:1.0:1787763747.902536:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763747.902545:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763747.902550:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763747.902553:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763747.902558:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763747.902562:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787763747.902567:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763747.902570:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787763747.902578:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763747.902582:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763747.902588:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763747.902594:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763747.902600:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763747.902603:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763747.902608:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763747.902615:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e580 x1874599832656000/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763772 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.902631:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763747.902638:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832656000/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763772 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.902653:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763747.902656:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763747.902663:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763747.902670:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763747.902676:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763747.902687:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316134933
00000100:00000001:1.0:1787763747.902691:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316136180
00000100:00000001:1.0:1787763747.902694:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316134933
00000100:00000001:1.0:1787763747.902696:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316136180
00000100:00000001:1.0:1787763747.902699:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763747.902702:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763747.902704:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763747.902710:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763747/real 1787763747]  req@ffff9751f876e580 x1874599832656000/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763747.902720:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763747.902722:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763747.902724:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763747.902727:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763747.902729:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763747.902732:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763747.902736:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832656000/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763747.902747:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832656000/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.902755:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763747.902757:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763747.902759:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763747.902760:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763747.902765:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832656000/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.902772:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763747.902777:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832656000/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.902786:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763747.902790:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763747.902792:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763747.902793:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763747.902796:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763747.902798:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763747.902801:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763747.902804:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763747.902807:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763747.902811:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832656000/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763747.902821:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832656000:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763747.902825:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763747.902830:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f0c0 x1874599832656128/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.902838:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763747.902839:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763747.902842:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763747.902845:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763747.902847:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763747.902851:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832656128:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763747.902855:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763747.902858:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763747.902860:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763747.902862:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763747.902867:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763747.902869:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763747.902874:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763747.902877:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763747.902881:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763747.902885:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832656128, portal 4
00000100:00000001:1.0:1787763747.902887:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763747.902889:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049530560 : -115096660021056 : ffff9751f876f0c0)
00000100:00000040:1.0:1787763747.902896:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f0c0 x1874599832656128/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763772 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.902905:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763747.902908:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763747.902910:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763747.902913:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832656128, offset 0
00000400:00000200:1.0:1787763747.902918:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763747.902922:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763747.902927:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763747.902931:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763747.902934:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763747.902937:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763747.902940:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763747.902944:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58361:58361:256:4294967295] 192.168.203.110@tcp LPNI seq info [58361:58361:8:4294967295]
00000400:00000200:1.0:1787763747.902949:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763747.902957:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763747.902964:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763747.902996:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763747.903020:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763747.903024:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787763747.903031:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763747.903035:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787763747.903041:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763747.903044:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763747.903048:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763747.903054:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763747.903058:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763747.903062:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763747.903082:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763747.903090:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f0c0 x1874599832656128/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763772 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.903103:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763747.903107:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f0c0 x1874599832656128/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763772 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.903116:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763747.903118:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763747.903123:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763747.903127:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763747.903148:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763747.903151:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316134950
00000100:00000001:1.0:1787763747.903154:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316136180
00000100:00000001:1.0:1787763747.903158:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316134950
00000100:00000001:1.0:1787763747.903161:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316136180
00000100:00000001:1.0:1787763747.903164:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763747.903167:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763747.903170:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763747.903176:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763747/real 1787763747]  req@ffff9751f876f0c0 x1874599832656128/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763747.903217:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763747.903219:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763747.903222:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763747.903236:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763747.903238:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763747.903240:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763747.903245:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f0c0 x1874599832656128/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763747.903253:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f0c0 x1874599832656128/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.903260:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763747.903262:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763747.903264:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763747.903265:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763747.903268:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f0c0 x1874599832656128/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.903274:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763747.903278:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f0c0 x1874599832656128/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.903285:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763747.903287:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763747.903289:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763747.903290:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763747.903292:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763747.903293:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763747.903295:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763747.903298:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763747.903300:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763747.903304:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f0c0 x1874599832656128/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763747.903311:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832656128:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763747.903315:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763747.903317:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763747.903320:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832655872/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.903326:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763747.903328:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763747.903330:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787763747.903333:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763747.903334:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763747.903336:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763747.903338:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763747.903340:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016800.
02000000:00000001:1.0:1787763747.903343:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763747.903345:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763747.903346:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:1.0:1787763747.903349:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763747.903351:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763747.903352:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763747.903355:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832656000/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.903362:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763747.903363:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763747.903364:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787763747.903366:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763747.903367:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763747.903368:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763747.903370:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763747.903371:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:1.0:1787763747.903373:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763747.903374:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763747.903375:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:1.0:1787763747.903377:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763747.903378:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763747.903380:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763747.903382:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f0c0 x1874599832656128/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763772 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763747.903388:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763747.903389:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763747.903390:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787763747.903392:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763747.903393:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763747.903394:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763747.903396:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763747.903398:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015c00.
02000000:00000001:1.0:1787763747.903399:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763747.903400:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763747.903402:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f0c0.
00000100:00000001:1.0:1787763747.903404:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763747.903405:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763747.903406:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763747.903410:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763747.903411:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763747.903413:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763747.903415:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763748.540299:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763748.540311:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763748.540316:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763748.540323:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787763748.540331:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763748.540337:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763748.540344:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763748.540348:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763748.540353:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763749.564319:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763749.564326:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763749.564329:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763749.564334:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763749.564340:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763749.564344:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763749.564348:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763749.564351:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763749.564356:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763750.588281:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763750.588296:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763750.588302:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763750.588311:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763750.588320:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763750.588328:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763750.588340:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763750.588347:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763750.588354:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763751.612272:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763751.612288:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763751.612298:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763751.612308:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763751.612315:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763751.612321:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763751.612327:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763751.612332:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763751.612338:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763752.636283:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763752.636292:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763752.636295:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763752.636303:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763752.636326:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763752.636332:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763752.636337:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763752.636342:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763752.636347:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763753.020310:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763753.020322:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787763753.020327:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763753.020332:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763753.020337:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763753.020345:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21467
00000400:00000200:1.0:1787763753.020353:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763753.020363:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763753.020367:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763753.020371:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763753.020374:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763753.020377:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763753.020381:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763753.020385:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763753.020387:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787763753.020390:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763753.020393:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787763753.020396:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787763753.020398:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787763753.020401:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763753.020404:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763753.020423:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763753.020427:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763753.020432:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787763753.020436:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763753.020440:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763753.020443:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763753.020448:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763753.020451:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763753.020454:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763753.020457:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763753.020467:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933840.
00000020:00000040:1.0:1787763753.020472:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787763753.020477:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763753.020479:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763753.020483:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763753.020486:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763753.020492:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763753.020504:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9c00.
00000100:00000001:1.0:1787763753.020512:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763753.020524:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933840 x1874599832656256/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763753.020539:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763753.020544:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763753.020550:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000083dd6d32] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933840 x1874599832656256/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763753.020594:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763753.020597:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763753.020601:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787763753.020603:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763753.020605:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763753.020608:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763753.020611:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21467
00000400:00000200:1.0:1787763753.020618:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763753.020621:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763753.020624:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763753.020627:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763753.020630:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763753.020632:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763753.020635:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763753.020639:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763753.020643:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787763753.020651:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763753.020655:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787763753.020664:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787763753.020672:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787763753.020688:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763753.020697:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763753.020707:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763753.020716:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763753.020722:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787763753.020732:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763753.020738:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763753.020742:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763753.020751:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763753.020756:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763753.020762:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763753.020769:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763753.020776:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763753.020781:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763753.020790:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932580.
00000020:00000040:1.0:1787763753.020795:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763753.020802:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763753.020808:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763753.020816:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763753.020821:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763753.020826:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763753.020833:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cac00.
00000100:00000001:1.0:1787763753.020843:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763753.020860:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932580 x1874599832656384/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763753.020888:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763753.020896:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763753.020911:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d026924] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932580 x1874599832656384/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763753.020966:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763753.020975:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763753.020981:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787763753.020987:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763753.020991:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763753.020997:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763753.021006:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21467
00000400:00000200:1.0:1787763753.021019:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763753.021030:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763753.021041:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763753.021053:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:2.1:1787763753.021061:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787763753.021064:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.1:1787763753.021066:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000100:00000040:1.0:1787763753.021074:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763753.021084:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763753.021091:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763753.021095:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787763753.021101:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763753.021107:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787763753.021112:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787763753.021119:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787763753.021128:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763753.021160:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763753.021163:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763753.021165:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763753.021169:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787763753.021172:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763753.021176:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763753.021179:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763753.021186:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763753.021198:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763753.021205:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763753.021210:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763753.021240:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763753.021251:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763753.021268:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930780.
00000020:00000040:1.0:1787763753.021279:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763753.021290:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763753.021297:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763753.021308:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763753.021310:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763753.021313:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763753.021322:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9400.
00000100:00000001:1.0:1787763753.021333:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763753.021349:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930780 x1874599832656512/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763753.021371:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763753.021380:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763753.021388:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3977950] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930780 x1874599832656512/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763753.021411:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763753.021420:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787763753.021429:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787763753.021433:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763753.021436:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787763753.021438:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763753.021446:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763753.021456:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763753.021466:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21467
00000400:00000200:1.0:1787763753.021480:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763753.021489:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763753.021499:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763753.021511:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763753.021518:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763753.021527:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763753.021534:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763753.021539:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763753.021541:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787763753.021545:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763753.021567:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787763753.021573:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787763753.021580:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:1.0:1787763753.021591:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763753.021598:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763753.021606:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763753.021609:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763753.021615:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:1.0:1787763753.021622:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763753.021649:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763753.021661:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763753.021671:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763753.021684:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763753.021694:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763753.021703:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1787763753.021716:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763753.021740:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763753.021754:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932940.
00000020:00000040:1.0:1787763753.021763:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787763753.021775:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763753.021781:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763753.021789:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763753.021795:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763753.021816:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763753.021827:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8400.
00000100:00000001:1.0:1787763753.021840:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763753.021858:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932940 x1874599832656640/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763753.021901:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763753.021911:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763753.021978:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000af90eaa] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932940 x1874599832656640/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763753.022000:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763753.022009:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21477)
00000800:00000200:2.1:1787763753.022202:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763753.022205:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787763753.022715:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763753.022719:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787763753.023123:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763753.023126:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:1.0:1787763753.660274:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763753.660286:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763753.660289:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763753.660297:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763753.660303:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763753.660308:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763753.660314:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763753.660332:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763753.660337:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763753.660394:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763753.660400:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763753.660404:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763754.684263:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763754.684273:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763754.684278:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763754.684284:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763754.684290:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763754.684296:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763754.684300:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763754.684305:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763754.684309:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763755.708230:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763755.708239:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763755.708243:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763755.708250:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763755.708258:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763755.708266:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763755.708282:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763755.708290:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763755.708302:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763756.732271:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763756.732281:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763756.732286:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763756.732296:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763756.732306:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763756.732315:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763756.732321:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763756.732324:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763756.732327:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763756.732386:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763756.732393:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763756.732398:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763757.756233:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763757.756244:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763757.756247:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763757.756256:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763757.756263:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763757.756267:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763757.756272:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763757.756276:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763757.756281:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763758.140277:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763758.140291:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787763758.140299:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763758.140307:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763758.140315:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763758.140330:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21472
00000400:00000200:1.0:1787763758.140346:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763758.140357:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763758.140361:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763758.140367:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763758.140373:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763758.140377:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763758.140381:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763758.140385:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763758.140388:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787763758.140392:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763758.140396:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787763758.140399:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787763758.140403:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787763758.140407:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763758.140410:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763758.140412:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763758.140414:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763758.140418:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787763758.140422:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763758.140425:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763758.140429:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763758.140433:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763758.140437:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763758.140439:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763758.140442:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763758.140451:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7931680.
00000020:00000040:1.0:1787763758.140455:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787763758.140461:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763758.140463:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763758.140467:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763758.140470:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763758.140475:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763758.140484:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8000.
00000100:00000001:1.0:1787763758.140493:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763758.140504:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7931680 x1874599832656768/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763758.140518:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763758.140523:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763758.140529:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d4255d6] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7931680 x1874599832656768/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763758.140557:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763758.140561:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763758.140565:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787763758.140567:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763758.140570:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763758.140573:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763758.140577:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21472
00000400:00000200:1.0:1787763758.140583:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763758.140588:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763758.140591:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763758.140596:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763758.140599:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763758.140603:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763758.140612:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763758.140617:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763758.140621:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787763758.140626:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763758.140629:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787763758.140636:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787763758.140640:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787763758.140652:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763758.140660:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763758.140665:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763758.140674:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763758.140682:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787763758.140691:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763758.140695:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763758.140701:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763758.140711:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763758.140717:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763758.140722:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763758.140728:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763758.140736:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763758.140744:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763758.140750:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930000.
00000020:00000040:1.0:1787763758.140756:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763758.140764:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763758.140767:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763758.140775:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763758.140777:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763758.140785:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763758.140790:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb400.
00000100:00000001:1.0:1787763758.140801:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763758.140818:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930000 x1874599832656896/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763758.140830:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763758.140835:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763758.140844:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6d90274] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930000 x1874599832656896/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763758.140879:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763758.140890:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763758.140894:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787763758.140922:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763758.140928:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763758.140933:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763758.140938:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21472
00000400:00000200:1.0:1787763758.140948:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763758.140955:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763758.140961:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763758.140970:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763758.140974:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763758.140982:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763758.140991:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763758.141001:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763758.141006:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787763758.141011:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763758.141017:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787763758.141024:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787763758.141031:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787763758.141040:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763758.141050:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763758.141055:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763758.141061:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763758.141065:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787763758.141076:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763758.141081:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763758.141085:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763758.141091:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763758.141098:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763758.141101:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763758.141104:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763758.141109:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763758.141116:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763758.141125:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933480.
00000020:00000040:1.0:1787763758.141163:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763758.141168:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763758.141170:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763758.141173:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763758.141175:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763758.141178:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763758.141184:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca000.
00000100:00000001:1.0:1787763758.141188:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763758.141202:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933480 x1874599832657024/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763758.141233:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763758.141241:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763758.141253:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3c2c33c] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933480 x1874599832657024/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763758.141285:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763758.141295:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787763758.141307:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787763758.141312:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763758.141321:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787763758.141325:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763758.141329:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763758.141339:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:2.1:1787763758.141346:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00080000:1.0:1787763758.141349:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21472
00000400:00000200:1.0:1787763758.141358:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763758.141363:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763758.141366:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763758.141370:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763758.141374:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763758.141382:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763758.141391:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763758.141401:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763758.141406:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787763758.141411:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763758.141415:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787763758.141422:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787763758.141427:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:1.0:1787763758.141438:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763758.141441:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763758.141446:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763758.141455:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763758.141458:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:1.0:1787763758.141461:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763758.141464:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763758.141467:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763758.141473:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763758.141477:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763758.141485:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763758.141490:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1787763758.141497:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763758.141500:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763758.141511:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930b40.
00000020:00000040:1.0:1787763758.141519:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787763758.141525:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763758.141529:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763758.141538:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763758.141540:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763758.141542:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763758.141549:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9000.
00000100:00000001:1.0:1787763758.141558:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763758.141574:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930b40 x1874599832657152/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763758.141591:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763758.141598:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763758.141616:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022d7f91a] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930b40 x1874599832657152/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763758.141646:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763758.141655:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21482)
00000800:00000200:2.1:1787763758.141891:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763758.142291:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763758.142614:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787763758.780278:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763758.780288:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763758.780294:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763758.780301:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763758.780311:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763758.780320:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763758.780330:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763758.780336:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763758.780343:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763759.804362:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763759.804372:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763759.804376:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763759.804383:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763759.804389:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763759.804395:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763759.804399:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763759.804404:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763759.804409:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763760.828291:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763760.828306:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763760.828314:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763760.828328:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763760.828342:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763760.828354:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763760.828363:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763760.828366:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763760.828371:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763761.213038:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763761.213051:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763761.213066:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763761.213091:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763758/real 1787763761]  req@ffff9751d7931680 x1874599832656768/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763761.213200:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763761.213207:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763761.213214:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0058.
00000400:00000200:1.0:1787763761.213231:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec7f8
00000400:00000010:1.0:1787763761.213241:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec7f8.
00000100:00000001:1.0:1787763761.213255:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763761.213277:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7931680 x1874599832656768/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763761.213328:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7931680 x1874599832656768/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763761.213360:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763761.213369:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763761.213380:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763761.213415:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763761.213437:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7931680 x1874599832656768/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763761.213467:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763761.213487:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7931680 x1874599832656768/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763761.213528:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763761.213544:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763761.213553:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763761.213560:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763761.213572:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763761.213580:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763761.213596:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763761.213615:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763761.213630:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763761.213652:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7931680 x1874599832656768/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763761.213677:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832656768:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763761.213693:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763761.213699:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763761.213708:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7931680 x1874599832656768/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763761.213723:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763761.213728:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763761.213736:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:1.0:1787763761.213745:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763761.213752:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763761.213756:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763761.213761:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763761.213766:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8000.
02000000:00000001:1.0:1787763761.213774:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763761.213778:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763761.213783:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7931680.
00000100:00000001:1.0:1787763761.213791:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763761.213794:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763761.213812:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763761.213817:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763761.213821:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763761.213824:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763761.213836:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763758/real 1787763761]  req@ffff9751d7930b40 x1874599832657152/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763761.213906:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763761.213910:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763761.213915:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0738.
00000400:00000200:1.0:1787763761.213924:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:1.0:1787763761.213928:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:1.0:1787763761.213936:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763761.213947:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930b40 x1874599832657152/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763761.213967:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930b40 x1874599832657152/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763761.213982:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763761.213985:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763761.213989:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763761.213992:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763761.214000:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930b40 x1874599832657152/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763761.214015:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763761.214022:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930b40 x1874599832657152/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763761.214036:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763761.214042:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763761.214047:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763761.214049:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763761.214053:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763761.214057:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763761.214063:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763761.214072:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763761.214078:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763761.214088:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930b40 x1874599832657152/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763761.214107:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832657152:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763761.214116:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763761.214126:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763758/real 1787763761]  req@ffff9751d7930000 x1874599832656896/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763761.214196:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763761.214198:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763761.214201:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0bb0.
00000400:00000200:1.0:1787763761.214207:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec440
00000400:00000010:1.0:1787763761.214210:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec440.
00000100:00000001:1.0:1787763761.214215:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763761.214222:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930000 x1874599832656896/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763761.214234:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930000 x1874599832656896/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763761.214244:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763761.214246:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763761.214249:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763761.214250:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763761.214255:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930000 x1874599832656896/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763761.214263:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763761.214269:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930000 x1874599832656896/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763761.214279:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763761.214284:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763761.214287:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763761.214289:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763761.214292:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763761.214319:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763761.214324:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763761.214330:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763761.214334:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763761.214342:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930000 x1874599832656896/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763761.214354:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832656896:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763761.214362:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763761.214368:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763758/real 1787763761]  req@ffff9751d7933480 x1874599832657024/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763761.214378:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763761.214380:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763761.214382:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da09a0.
00000400:00000200:1.0:1787763761.214385:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:1.0:1787763761.214388:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:1.0:1787763761.214392:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763761.214397:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933480 x1874599832657024/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763761.214410:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933480 x1874599832657024/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763761.214421:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763761.214425:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763761.214428:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763761.214430:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763761.214436:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933480 x1874599832657024/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763761.214455:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763761.214461:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933480 x1874599832657024/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763761.214469:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763761.214474:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763761.214478:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763761.214480:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763761.214482:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763761.214484:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763761.214487:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763761.214491:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763761.214495:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763761.214503:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933480 x1874599832657024/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763761.214512:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832657024:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763761.214517:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763761.214520:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763761.214527:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930b40 x1874599832657152/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763761.214535:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763761.214537:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763761.214541:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:1.0:1787763761.214546:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763761.214550:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763761.214552:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763761.214555:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763761.214559:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9000.
02000000:00000001:1.0:1787763761.214565:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763761.214569:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763761.214572:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930b40.
00000100:00000001:1.0:1787763761.214577:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763761.214580:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763761.214582:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763761.214588:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930000 x1874599832656896/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763761.214596:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763761.214598:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763761.214600:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:1.0:1787763761.214603:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763761.214604:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763761.214606:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763761.214608:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763761.214610:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787763761.214613:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763761.214615:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763761.214616:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930000.
00000100:00000001:1.0:1787763761.214619:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763761.214620:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763761.214622:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763761.214626:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933480 x1874599832657024/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763783 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763761.214633:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763761.214635:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763761.214636:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:1.0:1787763761.214639:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763761.214641:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763761.214642:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763761.214644:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763761.214646:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca000.
02000000:00000001:1.0:1787763761.214648:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763761.214650:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763761.214651:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933480.
00000100:00000001:1.0:1787763761.214657:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763761.214660:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763761.214664:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763761.214675:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763761.214678:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763761.214682:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763761.214685:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763761.852240:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763761.852251:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763761.852255:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763761.852263:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763761.852269:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763761.852274:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763761.852279:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763761.852289:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763761.852300:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763762.876275:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763762.876285:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763762.876292:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763762.876300:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763762.876308:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763762.876315:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763762.876319:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763762.876328:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763762.876333:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763763.260291:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763763.260305:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787763763.260309:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763763.260314:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763763.260321:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763763.260338:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21477
00000400:00000200:1.0:1787763763.260355:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763763.260367:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763763.260383:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763763.260400:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763763.260406:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763763.260411:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763763.260414:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763763.260419:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763763.260422:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787763763.260425:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763763.260429:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787763763.260432:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787763763.260435:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787763763.260439:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763763.260441:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763763.260444:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763763.260446:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763763.260449:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787763763.260454:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763763.260458:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763763.260461:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763763.260466:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763763.260470:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763763.260472:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763763.260476:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763763.260487:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933480.
00000020:00000040:1.0:1787763763.260491:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787763763.260496:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763763.260498:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763763.260502:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763763.260506:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763763.260511:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763763.260521:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca000.
00000100:00000001:1.0:1787763763.260531:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763763.260547:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933480 x1874599832657280/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763763.260567:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763763.260576:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763763.260586:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3c2c33c] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933480 x1874599832657280/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763763.260636:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763763.260641:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763763.260646:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787763763.260648:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763763.260651:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763763.260654:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763763.260659:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21477
00000400:00000200:1.0:1787763763.260665:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763763.260669:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763763.260672:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763763.260676:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763763.260679:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763763.260681:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763763.260695:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763763.260698:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763763.260700:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787763763.260703:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763763.260705:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787763763.260708:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787763763.260710:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787763763.260714:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763763.260716:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763763.260719:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763763.260723:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763763.260728:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787763763.260734:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763763.260743:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763763.260747:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763763.260756:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763763.260761:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763763.260764:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763763.260771:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763763.260776:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763763.260780:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763763.260788:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930000.
00000020:00000040:1.0:1787763763.260792:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763763.260797:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763763.260800:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763763.260803:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763763.260806:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763763.260810:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763763.260818:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb400.
00000100:00000001:1.0:1787763763.260827:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763763.260840:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930000 x1874599832657408/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763763.260857:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763763.260891:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763763.260908:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6d90274] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930000 x1874599832657408/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763763.260939:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763763.260951:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763763.260968:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787763763.260977:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763763.260983:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763763.260990:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763763.261002:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21477
00000400:00000200:1.0:1787763763.261017:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763763.261025:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763763.261030:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763763.261036:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763763.261045:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763763.261055:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763763.261063:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763763.261073:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763763.261079:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787763763.261086:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763763.261094:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787763763.261104:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787763763.261114:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787763763.261130:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763763.261182:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763763.261187:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763763.261191:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763763.261197:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787763763.261204:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763763.261213:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763763.261219:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763763.261227:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763763.261239:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763763.261247:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763763.261252:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763763.261262:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763763.261271:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763763.261288:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930b40.
00000020:00000040:1.0:1787763763.261295:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763763.261304:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763763.261309:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763763.261322:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763763.261330:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763763.261338:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763763.261352:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9000.
00000100:00000001:1.0:1787763763.261364:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763763.261388:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930b40 x1874599832657536/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763763.261426:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763763.261437:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763763.261453:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022d7f91a] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930b40 x1874599832657536/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763763.261485:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763763.261494:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787763763.261514:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787763763.261524:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763763.261537:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787763763.261543:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763763.261551:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763763.261557:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763763.261565:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21477
00000400:00000200:1.0:1787763763.261574:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763763.261594:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763763.261606:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763763.261620:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763763.261631:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763763.261643:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763763.261654:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763763.261664:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763763.261668:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787763763.261674:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763763.261682:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787763763.261690:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787763763.261700:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:1.0:1787763763.261715:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763763.261725:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763763.261731:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763763.261740:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763763.261745:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:1.0:1787763763.261760:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763763.261771:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763763.261780:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:2.1:1787763763.261781:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763763.261789:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
02000000:00000001:1.0:1787763763.261792:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763763.261806:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763763.261816:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763763.261826:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1787763763.261833:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763763.261844:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763763.261855:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7931680.
00000020:00000040:1.0:1787763763.261890:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787763763.261901:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763763.261909:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763763.261914:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763763.261919:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763763.261927:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763763.261938:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8000.
00000100:00000001:1.0:1787763763.261948:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763763.261964:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7931680 x1874599832657664/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763763.261982:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763763.261990:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763763.262006:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d4255d6] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7931680 x1874599832657664/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763763.262065:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763763.262078:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21487)
00000800:00000200:2.0:1787763763.262795:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763763.262800:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787763763.262806:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763763.262810:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787763763.262815:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763763.262818:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
00000400:00000200:2.0:1787763763.262822:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763763.262827:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763763.262832:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec990
00000400:00000010:2.0:1787763763.262834:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec990.
00000100:00000001:2.0:1787763763.262838:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763763.262844:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930000 x1874599832657408/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763788 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.262855:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763763.262859:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930000 x1874599832657408/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763788 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.262895:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763763.262897:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763763.262903:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763763.262909:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787763763.262913:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763763.262916:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316150305
00000100:00000001:2.0:1787763763.262920:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316151540
00000100:00000001:2.0:1787763763.262923:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316150305
00000100:00000001:2.0:1787763763.262926:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316151540
00000100:00000001:2.0:1787763763.262928:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763763.262931:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763763.262934:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763763.262939:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763763/real 1787763763]  req@ffff9751d7930000 x1874599832657408/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763763.262949:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763763.262951:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763763.262955:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787763763.262958:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000400:00000010:2.0:1787763763.262960:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000100:00000001:2.0:1787763763.262963:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763763.262967:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930000 x1874599832657408/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763763.262977:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930000 x1874599832657408/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.262985:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763763.262987:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763763.262989:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763763.262991:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763763.262995:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930000 x1874599832657408/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.263003:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763763.263008:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930000 x1874599832657408/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.263017:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763763.263020:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763763.263023:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763763.263024:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763763.263027:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763763.263040:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763763.263043:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763763.263047:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763763.263050:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763763.263055:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930000 x1874599832657408/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763763.263065:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832657408:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763763.263072:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763763.263076:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930b40 x1874599832657536/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.263085:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763763.263087:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763763.263089:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763763.263091:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763763.263094:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763763.263097:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832657536:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763763.263101:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763763.263104:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763763.263106:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763763.263109:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763763.263113:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763763.263116:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763763.263120:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763763.263123:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787763763.263127:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ece58.
00000100:00000200:2.0:1787763763.263131:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832657536, portal 10
00000100:00000001:2.0:1787763763.263162:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763763.263164:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497724224 : -115097211827392 : ffff9751d7930b40)
00000100:00000040:2.0:1787763763.263173:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930b40 x1874599832657536/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763788 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.263181:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763763.263184:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763763.263188:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec990.
00000100:00000200:2.0:1787763763.263191:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832657536, offset 0
00000400:00000200:2.0:1787763763.263196:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763763.263202:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763763.263207:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763763.263211:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763763.263215:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763763.263218:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763763.263221:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763763.263225:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58372:58372:256:4294967295] 192.168.203.110@tcp LPNI seq info [58372:58372:8:4294967295]
00000400:00000200:2.0:1787763763.263230:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763763.263239:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763763.263244:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763763.263248:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f400.
00000800:00000200:2.0:1787763763.263253:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763763.263256:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787763763.263262:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763763.263266:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787763763.263271:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763763.263273:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
00000400:00000200:2.0:1787763763.263277:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763763.263282:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763763.263286:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec990
00000400:00000010:2.0:1787763763.263288:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec990.
00000100:00000001:2.0:1787763763.263291:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763763.263295:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930b40 x1874599832657536/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763788 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.263305:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763763.263309:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930b40 x1874599832657536/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763788 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.263317:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763763.263320:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763763.263323:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763763.263327:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787763763.263331:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763763.263333:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316150294
00000100:00000001:2.0:1787763763.263337:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316151540
00000100:00000001:2.0:1787763763.263339:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316150294
00000100:00000001:2.0:1787763763.263343:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316151540
00000100:00000001:2.0:1787763763.263346:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763763.263349:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763763.263351:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763763.263356:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763763/real 1787763763]  req@ffff9751d7930b40 x1874599832657536/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763763.263365:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763763.263366:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763763.263368:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787763763.263371:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000400:00000010:2.0:1787763763.263373:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000100:00000001:2.0:1787763763.263375:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763763.263379:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930b40 x1874599832657536/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763763.263389:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930b40 x1874599832657536/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.263397:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763763.263399:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763763.263401:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763763.263403:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763763.263407:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930b40 x1874599832657536/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.263415:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763763.263419:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930b40 x1874599832657536/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.263427:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763763.263430:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763763.263432:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763763.263433:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763763.263436:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763763.263439:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763763.263442:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763763.263446:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763763.263449:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763763.263454:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930b40 x1874599832657536/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763763.263463:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832657536:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763763.263467:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763763.263472:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7931680 x1874599832657664/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.263479:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763763.263481:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763763.263484:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763763.263486:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763763.263488:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763763.263491:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832657664:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763763.263495:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763763.263498:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763763.263500:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763763.263503:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763763.263507:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763763.263509:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763763.263513:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763763.263516:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787763763.263519:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ece58.
00000100:00000200:2.0:1787763763.263523:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832657664, portal 4
00000100:00000001:2.0:1787763763.263526:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763763.263528:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497727104 : -115097211824512 : ffff9751d7931680)
00000100:00000040:2.0:1787763763.263534:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7931680 x1874599832657664/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763788 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.263542:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763763.263545:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763763.263548:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec990.
00000100:00000200:2.0:1787763763.263551:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832657664, offset 0
00000400:00000200:2.0:1787763763.263555:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763763.263559:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763763.263563:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763763.263566:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763763.263570:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763763.263572:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763763.263575:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763763.263579:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58373:58373:256:4294967295] 192.168.203.110@tcp LPNI seq info [58373:58373:8:4294967295]
00000400:00000200:2.0:1787763763.263584:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763763.263592:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763763.263597:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763763.263600:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f400.
00000800:00000200:2.0:1787763763.263604:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763763.263607:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787763763.263612:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763763.263615:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787763763.263620:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763763.263622:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
00000400:00000200:2.0:1787763763.263626:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763763.263630:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763763.263633:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec990
00000400:00000010:2.0:1787763763.263635:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec990.
00000100:00000001:2.0:1787763763.263639:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763763.263646:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7931680 x1874599832657664/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763788 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.263663:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763763.263671:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7931680 x1874599832657664/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763788 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.263697:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763763.263700:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763763.263707:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763763.263713:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787763763.263719:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763763.263723:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316150311
00000100:00000001:2.0:1787763763.263728:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316151540
00000100:00000001:2.0:1787763763.263734:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316150311
00000100:00000001:2.0:1787763763.263736:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316151540
00000100:00000001:2.0:1787763763.263739:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763763.263742:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763763.263745:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763763.263751:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763763/real 1787763763]  req@ffff9751d7931680 x1874599832657664/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763763.263761:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763763.263763:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763763.263766:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787763763.263769:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000400:00000010:2.0:1787763763.263771:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000100:00000001:2.0:1787763763.263774:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763763.263778:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7931680 x1874599832657664/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763763.263789:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7931680 x1874599832657664/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.263797:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763763.263799:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763763.263801:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763763.263803:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763763.263807:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7931680 x1874599832657664/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.263816:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763763.263821:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7931680 x1874599832657664/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.263830:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763763.263833:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763763.263835:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763763.263836:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763763.263838:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763763.263840:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763763.263844:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763763.263847:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763763.263850:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763763.263854:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7931680 x1874599832657664/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763763.263887:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832657664:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763763.263892:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763763.263895:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763763.263899:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930000 x1874599832657408/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.263908:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763763.263910:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763763.263913:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763763.263916:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763763.263919:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763763.263921:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763763.263923:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763763.263926:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb400.
02000000:00000001:2.0:1787763763.263930:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763763.263932:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763763.263934:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930000.
00000100:00000001:2.0:1787763763.263937:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763763.263939:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763763.263941:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763763.263945:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930b40 x1874599832657536/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.263953:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763763.263955:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763763.263956:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763763.263959:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763763.263960:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763763.263962:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763763.263964:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763763.263966:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9000.
02000000:00000001:2.0:1787763763.263968:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763763.263969:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763763.263971:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930b40.
00000100:00000001:2.0:1787763763.263973:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763763.263975:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763763.263977:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763763.263980:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7931680 x1874599832657664/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763788 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763763.263988:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763763.263989:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763763.263991:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763763.263993:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763763.263994:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763763.263996:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787763763.263998:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763763.263999:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8000.
02000000:00000001:2.0:1787763763.264002:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763763.264003:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763763.264005:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7931680.
00000100:00000001:2.0:1787763763.264008:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763763.264010:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763763.264013:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763763.264017:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763763.264019:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763763.264022:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763763.264024:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763763.900247:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763763.900258:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763763.900261:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763763.900270:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763763.900278:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763763.900286:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763763.900294:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763763.900299:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763763.900306:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763763.900433:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787763763.900442:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787763763.900450:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:2.0:1787763764.540360:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316151540, at 4316151552
00000100:00000010:2.0:1787763764.540385:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00000100:00000001:1.0:1787763764.924240:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763764.924253:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763764.924258:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763764.924265:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763764.924271:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763764.924286:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763764.924304:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763764.924311:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763764.924336:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763764.924406:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787763764.924411:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787763764.924415:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763765.948281:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763765.948293:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763765.948299:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763765.948310:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763765.948325:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763765.948342:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763765.948350:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763765.948358:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763765.948364:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763765.948431:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787763765.948436:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787763765.948439:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763766.972221:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763766.972232:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763766.972236:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763766.972244:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763766.972249:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763766.972256:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763766.972263:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763766.972269:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763766.972282:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763766.972371:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787763766.972376:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787763766.972380:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763767.996267:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763767.996277:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763767.996280:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763767.996296:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763767.996315:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763767.996322:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763767.996330:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763767.996333:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763767.996337:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763767.996395:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787763767.996401:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787763767.996404:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787763768.380312:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763768.380327:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787763768.380330:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763768.380333:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763768.380337:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763768.380344:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21482
00000400:00000200:1.0:1787763768.380351:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763768.380359:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763768.380362:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763768.380365:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763768.380368:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763768.380371:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763768.380374:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763768.380377:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763768.380378:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787763768.380381:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763768.380384:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787763768.380385:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787763768.380387:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787763768.380389:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763768.380390:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763768.380392:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763768.380394:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763768.380396:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787763768.380401:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763768.380403:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763768.380405:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763768.380409:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763768.380411:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763768.380413:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763768.380415:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763768.380423:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7931680.
00000020:00000040:1.0:1787763768.380426:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787763768.380430:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763768.380432:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763768.380435:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763768.380437:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763768.380441:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763768.380450:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8800.
00000100:00000001:1.0:1787763768.380456:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763768.380464:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7931680 x1874599832657792/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763768.380472:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763768.380476:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763768.380481:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d4255d6] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7931680 x1874599832657792/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763768.380501:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763768.380503:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763768.380506:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787763768.380507:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763768.380508:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763768.380510:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763768.380513:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21482
00000400:00000200:1.0:1787763768.380519:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763768.380524:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763768.380527:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763768.380534:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763768.380536:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763768.380539:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763768.380543:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763768.380543:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763768.380550:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:1.0:1787763768.380551:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1787763768.380553:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:1.0:1787763768.380555:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787763768.380560:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763768.380565:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000040:2.0:1787763768.380566:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7931680 x1874599832657792/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1787763768.380568:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787763768.380578:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787763768.380594:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763768.380595:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763768.380598:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763768.380601:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1787763768.380603:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763768.380607:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1787763768.380611:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:2.0:1787763768.380613:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763768.380620:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00100000:2.0:1787763768.380621:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832657792:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787763768.380633:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:1.0:1787763768.380634:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:2.0:1787763768.380639:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763768.380642:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763768.380644:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763768.380648:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763768.380648:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:1.0:1787763768.380655:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:2.0:1787763768.380658:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:1.0:1787763768.380660:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763768.380662:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763768.380665:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763768.380669:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:2.0:1787763768.380673:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000008:00100000:1.0:1787763768.380674:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000010:2.0:1787763768.380679:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000008:00000001:1.0:1787763768.380681:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763768.380686:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763768.380688:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ece58.
00000100:00000010:1.0:1787763768.380694:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930b40.
00000100:00000200:2.0:1787763768.380697:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832657792, portal 25
00000020:00000040:1.0:1787763768.380699:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000100:00000001:2.0:1787763768.380701:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763768.380706:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497727104 : -115097211824512 : ffff9751d7931680)
02000000:00000001:1.0:1787763768.380710:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763768.380714:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763768.380716:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7931680 x1874599832657792/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763793 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763768.380720:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763768.380726:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763768.380732:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763768.380732:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763768.380744:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
02000000:00000010:1.0:1787763768.380745:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb800.
00000400:00000010:2.0:1787763768.380751:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec990.
00000100:00000001:1.0:1787763768.380752:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763768.380764:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832657792, offset 0
00000100:00100000:1.0:1787763768.380767:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930b40 x1874599832657920/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763768.380774:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763768.380776:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763768.380788:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000040:1.0:1787763768.380789:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022d7f91a] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930b40 x1874599832657920/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787763768.380803:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763768.380804:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763768.380811:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763768.380815:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:2.0:1787763768.380817:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:1.0:1787763768.380820:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763768.380822:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1787763768.380824:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1787763768.380844:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763768.380850:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21482
00000400:00000200:2.0:1787763768.380851:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763768.380866:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763768.380867:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763768.380880:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763768.380881:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763768.380887:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1787763768.380898:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58374:58374:256:4294967295] 192.168.203.110@tcp LPNI seq info [58374:58374:8:4294967295]
00000100:00000040:1.0:1787763768.380903:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763768.380907:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1787763768.380909:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:1.0:1787763768.380914:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763768.380917:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763768.380920:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763768.380923:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787763768.380927:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763768.380931:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000400:00000200:2.0:1787763768.380933:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000020:00000001:1.0:1787763768.380937:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787763768.380942:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:2.0:1787763768.380947:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000040:1.0:1787763768.380948:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763768.380958:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763768.380961:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000010:2.0:1787763768.380961:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f400.
00000100:00000001:1.0:1787763768.380970:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763768.380972:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000800:00000200:2.0:1787763768.380979:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00080000:1.0:1787763768.380981:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763768.380990:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787763768.380990:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
02000000:00000001:1.0:1787763768.380996:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763768.381006:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:2.1:1787763768.381008:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:1.0:1787763768.381010:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763768.381016:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763768.381019:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000800:00000200:2.1:1787763768.381019:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000008:00000001:1.0:1787763768.381025:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763768.381028:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763768.381032:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930000.
00000800:00000100:2.0:1787763768.381036:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000020:00000040:1.0:1787763768.381037:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763768.381047:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000010:2.0:1787763768.381048:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
02000000:00000001:1.0:1787763768.381049:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763768.381062:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763768.381064:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1787763768.381064:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:1.0:1787763768.381069:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763768.381073:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
02000000:00000010:1.0:1787763768.381075:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8c00.
00000400:00000200:2.0:1787763768.381079:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec990
00000100:00000001:1.0:1787763768.381080:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763768.381087:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec990.
00000100:00100000:1.0:1787763768.381092:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930000 x1874599832658048/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763768.381094:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763768.381106:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7931680 x1874599832657792/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763793 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763768.381114:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763768.381121:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763768.381162:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763768.381165:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6d90274] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930000 x1874599832658048/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787763768.381171:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7931680 x1874599832657792/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763793 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763768.381179:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763768.381189:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787763768.381200:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787763768.381205:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763768.381205:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763768.381217:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:1.0:1787763768.381218:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787763768.381224:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763768.381225:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763768.381227:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763768.381230:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21482
00000100:00000001:2.0:1787763768.381233:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763768.381238:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763768.381247:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1787763768.381248:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:1.0:1787763768.381253:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787763768.381256:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316156660
00000100:00000040:1.0:1787763768.381262:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000010:2.0:1787763768.381266:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00000001:1.0:1787763768.381267:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763768.381274:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763768.381277:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763768.381278:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000020:00000001:1.0:1787763768.381284:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763768.381286:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787763768.381288:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763768.381290:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787763768.381293:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787763768.381295:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:2.0:1787763768.381297:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316156660
00000100:00000040:1.0:1787763768.381298:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763768.381302:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763768.381307:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000010:2.0:1787763768.381308:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7120.
00000100:00000001:1.0:1787763768.381314:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763768.381317:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000020:00000040:1.0:1787763768.381318:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:2.0:1787763768.381321:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:1.0:1787763768.381323:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763768.381329:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763768.381329:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763768.381344:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787763768.381345:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000001:1.0:1787763768.381357:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:2.0:1787763768.381358:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763768/real 1787763768]  req@ffff9751d7931680 x1874599832657792/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787763768.381367:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763768.381375:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763768.381381:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000400:00000001:2.0:1787763768.381383:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000008:00000001:1.0:1787763768.381388:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:2.0:1787763768.381388:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763768.381396:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000100:00000001:1.0:1787763768.381397:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763768.381411:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933480.
00000400:00000200:2.0:1787763768.381411:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000400:00000010:2.0:1787763768.381418:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000020:00000040:1.0:1787763768.381419:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787763768.381429:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1787763768.381430:0:257575:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:1.0:1787763768.381432:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763768.381435:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763768.381436:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763768.381438:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763768.381442:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7931680 x1874599832657792/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1787763768.381443:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca400.
00000100:00000001:1.0:1787763768.381448:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763768.381456:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933480 x1874599832658176/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763768.381476:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:2.0:1787763768.381481:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7931680 x1874599832657792/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763768.381482:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763768.381498:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3c2c33c] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933480 x1874599832658176/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.381504:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763768.381510:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763768.381515:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763768.381515:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763768.381518:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763768.381528:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21493)
00000100:00100000:2.0:1787763768.381528:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7931680 x1874599832657792/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.381542:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763768.381547:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7931680 x1874599832657792/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.381552:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763768.381554:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763768.381555:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763768.381556:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763768.381559:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763768.381562:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763768.381564:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763768.381568:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763768.381570:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763768.381573:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7931680 x1874599832657792/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763768.381580:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832657792:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787763768.381583:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763768.381584:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763768.381586:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7931680 x1874599832657792/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.381596:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763768.381599:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763768.381601:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763768.381604:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763768.381606:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763768.381608:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787763768.381609:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763768.381612:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8800.
02000000:00000001:2.0:1787763768.381615:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763768.381617:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763768.381618:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7931680.
00000100:00000001:2.0:1787763768.381621:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763768.381622:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763768.381624:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763768.381630:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763768.381632:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763768.381633:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763768.381635:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763768.381636:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763768.381639:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930b40 x1874599832657920/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.381644:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763768.381645:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763768.381647:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763768.381648:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763768.381650:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763768.381652:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832657920:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763768.381654:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763768.381656:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763768.381657:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763768.381659:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763768.381661:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763768.381663:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763768.381666:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763768.381668:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787763768.381671:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ece58.
00000100:00000200:2.0:1787763768.381674:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832657920, portal 10
00000100:00000001:2.0:1787763768.381675:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763768.381676:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497724224 : -115097211827392 : ffff9751d7930b40)
00000100:00000040:2.0:1787763768.381682:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930b40 x1874599832657920/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763793 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.381686:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763768.381688:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763768.381689:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec990.
00000100:00000200:2.0:1787763768.381691:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832657920, offset 0
00000400:00000200:2.0:1787763768.381694:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763768.381697:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763768.381699:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763768.381702:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763768.381704:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763768.381706:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763768.381707:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763768.381710:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58375:58375:256:4294967295] 192.168.203.110@tcp LPNI seq info [58375:58375:8:4294967295]
00000400:00000200:2.0:1787763768.381712:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763768.381717:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763768.381721:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763768.381723:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f400.
00000800:00000200:2.0:1787763768.381726:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763768.381728:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:2.1:1787763768.381732:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763768.381734:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:2.0:1787763768.381737:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763768.381738:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
00000400:00000200:2.0:1787763768.381740:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763768.381743:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763768.381745:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec990
00000400:00000010:2.0:1787763768.381746:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec990.
00000100:00000001:2.0:1787763768.381749:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763768.381751:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930b40 x1874599832657920/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763793 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.381756:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763768.381758:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930b40 x1874599832657920/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763793 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.381763:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763768.381764:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763768.381767:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763768.381770:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787763768.381772:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763768.381773:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316155424
00000100:00000001:2.0:1787763768.381776:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316156660
00000100:00000001:2.0:1787763768.381778:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316155424
00000100:00000001:2.0:1787763768.381779:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316156660
00000100:00000001:2.0:1787763768.381781:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763768.381782:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763768.381784:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763768.381786:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763768/real 1787763768]  req@ffff9751d7930b40 x1874599832657920/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763768.381790:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763768.381791:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763768.381792:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787763768.381793:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000400:00000010:2.0:1787763768.381794:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000100:00000001:2.0:1787763768.381796:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763768.381798:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930b40 x1874599832657920/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763768.381803:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930b40 x1874599832657920/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.381812:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763768.381813:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763768.381814:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763768.381815:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763768.381818:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930b40 x1874599832657920/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.381822:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763768.381824:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930b40 x1874599832657920/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.381847:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763768.381848:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763768.381849:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763768.381850:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763768.381851:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763768.381861:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763768.381863:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763768.381865:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763768.381867:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763768.381870:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930b40 x1874599832657920/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763768.381876:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832657920:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763768.381878:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763768.381880:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930000 x1874599832658048/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.381885:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763768.381886:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763768.381888:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763768.381889:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763768.381890:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763768.381892:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832658048:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763768.381894:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763768.381896:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763768.381897:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763768.381898:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763768.381901:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763768.381902:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763768.381904:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763768.381906:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787763768.381909:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ece58.
00000100:00000200:2.0:1787763768.381917:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832658048, portal 10
00000100:00000001:2.0:1787763768.381919:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763768.381920:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497721344 : -115097211830272 : ffff9751d7930000)
00000100:00000040:2.0:1787763768.381927:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930000 x1874599832658048/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763793 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.381932:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763768.381934:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763768.381935:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec990.
00000100:00000200:2.0:1787763768.381937:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832658048, offset 0
00000400:00000200:2.0:1787763768.381939:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763768.381942:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763768.381944:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763768.381946:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763768.381948:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763768.381950:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763768.381952:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763768.381954:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58376:58376:256:4294967295] 192.168.203.110@tcp LPNI seq info [58376:58376:8:4294967295]
00000400:00000200:2.0:1787763768.381957:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763768.381961:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763768.381965:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763768.381967:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f400.
00000800:00000200:2.0:1787763768.381970:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763768.381972:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:2.1:1787763768.381975:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763768.381977:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:2.0:1787763768.381980:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763768.381981:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
00000400:00000200:2.0:1787763768.381984:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763768.381986:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763768.381988:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec990
00000400:00000010:2.0:1787763768.381990:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec990.
00000100:00000001:2.0:1787763768.381991:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763768.381994:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930000 x1874599832658048/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763793 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.382002:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763768.382005:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930000 x1874599832658048/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763793 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.382012:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763768.382014:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763768.382018:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763768.382021:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787763768.382025:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763768.382027:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316155412
00000100:00000001:2.0:1787763768.382030:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316156660
00000100:00000001:2.0:1787763768.382032:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316155412
00000100:00000001:2.0:1787763768.382035:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316156660
00000100:00000001:2.0:1787763768.382038:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763768.382040:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763768.382042:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763768.382046:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763768/real 1787763768]  req@ffff9751d7930000 x1874599832658048/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763768.382055:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763768.382056:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763768.382057:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787763768.382059:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000400:00000010:2.0:1787763768.382060:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000100:00000001:2.0:1787763768.382062:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763768.382064:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930000 x1874599832658048/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763768.382070:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930000 x1874599832658048/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.382074:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763768.382075:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763768.382076:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763768.382077:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763768.382079:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930000 x1874599832658048/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.382084:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763768.382086:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930000 x1874599832658048/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.382091:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763768.382093:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763768.382094:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763768.382094:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763768.382096:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763768.382097:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763768.382098:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763768.382100:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763768.382101:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763768.382104:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930000 x1874599832658048/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763768.382110:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832658048:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763768.382112:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763768.382115:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933480 x1874599832658176/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.382119:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763768.382120:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763768.382121:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763768.382122:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763768.382123:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763768.382125:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832658176:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763768.382127:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763768.382128:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763768.382130:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763768.382131:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763768.382156:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787763768.382158:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763768.382161:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763768.382163:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787763768.382166:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ece58.
00000100:00000200:2.0:1787763768.382168:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832658176, portal 4
00000100:00000001:2.0:1787763768.382170:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763768.382171:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497734784 : -115097211816832 : ffff9751d7933480)
00000100:00000040:2.0:1787763768.382176:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933480 x1874599832658176/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763793 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.382180:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763768.382182:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763768.382183:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec990.
00000100:00000200:2.0:1787763768.382185:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832658176, offset 0
00000400:00000200:2.0:1787763768.382188:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763768.382191:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763768.382193:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763768.382195:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763768.382197:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763768.382199:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763768.382201:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763768.382203:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58377:58377:256:4294967295] 192.168.203.110@tcp LPNI seq info [58377:58377:8:4294967295]
00000400:00000200:2.0:1787763768.382206:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763768.382210:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763768.382213:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763768.382215:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f400.
00000800:00000200:2.0:1787763768.382217:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763768.382219:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:2.1:1787763768.382222:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763768.382225:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:2.0:1787763768.382227:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763768.382229:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
00000400:00000200:2.0:1787763768.382231:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763768.382233:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763768.382236:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec990
00000400:00000010:2.0:1787763768.382237:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec990.
00000100:00000001:2.0:1787763768.382238:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763768.382240:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933480 x1874599832658176/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763793 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.382246:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763768.382248:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933480 x1874599832658176/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763793 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.382253:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763768.382254:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763768.382256:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763768.382258:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787763768.382260:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763768.382262:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316155430
00000100:00000001:2.0:1787763768.382264:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316156660
00000100:00000001:2.0:1787763768.382265:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316155430
00000100:00000001:2.0:1787763768.382267:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316156660
00000100:00000001:2.0:1787763768.382268:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763768.382269:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763768.382271:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763768.382273:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763768/real 1787763768]  req@ffff9751d7933480 x1874599832658176/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763768.382277:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763768.382278:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763768.382279:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787763768.382281:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000400:00000010:2.0:1787763768.382282:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000100:00000001:2.0:1787763768.382283:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763768.382286:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933480 x1874599832658176/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763768.382291:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933480 x1874599832658176/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.382294:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763768.382295:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763768.382296:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763768.382297:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763768.382299:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933480 x1874599832658176/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.382303:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763768.382305:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933480 x1874599832658176/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.382309:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763768.382311:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763768.382312:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763768.382313:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763768.382315:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763768.382316:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763768.382318:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763768.382320:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763768.382321:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763768.382323:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933480 x1874599832658176/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763768.382328:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832658176:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763768.382330:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763768.382331:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763768.382333:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930b40 x1874599832657920/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.382337:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763768.382338:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763768.382339:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763768.382341:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763768.382342:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763768.382343:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763768.382344:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763768.382346:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb800.
02000000:00000001:2.0:1787763768.382348:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763768.382349:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763768.382349:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930b40.
00000100:00000001:2.0:1787763768.382351:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763768.382352:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763768.382353:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763768.382355:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930000 x1874599832658048/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.382364:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763768.382365:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763768.382366:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763768.382368:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763768.382368:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763768.382369:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763768.382370:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763768.382371:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8c00.
02000000:00000001:2.0:1787763768.382372:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763768.382373:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763768.382374:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930000.
00000100:00000001:2.0:1787763768.382375:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763768.382376:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763768.382376:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763768.382378:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933480 x1874599832658176/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763793 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763768.382382:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763768.382383:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763768.382383:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787763768.382385:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763768.382385:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763768.382386:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787763768.382387:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763768.382388:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca400.
02000000:00000001:2.0:1787763768.382389:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763768.382390:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763768.382391:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933480.
00000100:00000001:2.0:1787763768.382393:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763768.382394:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763768.382395:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763768.382398:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763768.382399:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763768.382400:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763768.382401:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763769.020328:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763769.020342:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763769.020347:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763769.020357:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763769.020364:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763769.020372:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763769.020380:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763769.020388:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763769.020401:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763769.020484:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763769.020491:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763769.020495:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:2.0:1787763769.660252:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316156660, at 4316156672
00000100:00000010:2.0:1787763769.660262:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00000100:00000001:1.0:1787763770.044238:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763770.044253:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763770.044265:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763770.044281:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763770.044291:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763770.044297:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763770.044300:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763770.044303:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763770.044306:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763770.044356:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787763770.044360:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787763770.044374:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763771.068223:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763771.068231:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763771.068234:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763771.068241:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763771.068250:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763771.068256:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763771.068261:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763771.068266:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763771.068271:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763771.068344:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787763771.068348:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787763771.068350:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763772.092225:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763772.092233:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763772.092236:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763772.092242:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763772.092246:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763772.092250:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763772.092254:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763772.092257:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763772.092260:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763772.092327:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763772.092333:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763772.092337:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763773.116290:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763773.116299:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763773.116309:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763773.116317:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763773.116340:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763773.116344:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.116349:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763773.116352:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.116357:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763773.116372:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763773.116381:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763773.116389:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787763773.500261:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763773.500270:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787763773.500275:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763773.500280:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763773.500285:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763773.500294:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21488
00000400:00000200:2.0:1787763773.500303:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763773.500310:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763773.500313:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763773.500316:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763773.500319:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763773.500321:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763773.500330:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763773.500334:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763773.500336:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787763773.500339:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763773.500355:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787763773.500357:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787763773.500359:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787763773.500362:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763773.500363:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763773.500364:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763773.500365:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763773.500368:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787763773.500371:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763773.500373:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763773.500377:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763773.500381:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763773.500383:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763773.500385:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787763773.500387:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763773.500398:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658780.
00000020:00000040:2.0:1787763773.500401:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787763773.500405:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763773.500408:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763773.500414:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763773.500418:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763773.500422:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763773.500431:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5b9800.
00000100:00000001:2.0:1787763773.500439:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763773.500449:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658780 x1874599832658304/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763773.500458:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763773.500462:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763773.500466:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000744875b5] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658780 x1874599832658304/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763773.500488:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763773.500492:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763773.500501:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787763773.500503:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763773.500504:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763773.500509:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763773.500514:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21488
00000400:00000200:2.0:1787763773.500520:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763773.500525:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763773.500528:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763773.500533:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763773.500536:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787763773.500539:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1787763773.500542:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763773.500551:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787763773.500555:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1787763773.500556:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:1.0:1787763773.500560:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:2.0:1787763773.500562:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:2.0:1787763773.500566:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763773.500569:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:2.0:1787763773.500571:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:2.0:1787763773.500581:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:2.0:1787763773.500594:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000040:1.0:1787763773.500602:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832658304/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763773.500603:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763773.500612:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763773.500623:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763773.500634:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:2.0:1787763773.500649:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763773.500662:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763773.500670:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787763773.500676:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763773.500680:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763773.500681:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763773.500689:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763773.500691:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1787763773.500693:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1787763773.500697:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00100000:2.0:1787763773.500700:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1787763773.500704:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1787763773.500708:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763773.500717:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763773.500719:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832658304:192.168.203.110@tcp:250:kworker.0
00000100:00000010:2.0:1787763773.500733:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000100:00000001:1.0:1787763773.500734:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:2.0:1787763773.500741:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763773.500743:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763773.500746:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763773.500750:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763773.500751:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763773.500768:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763773.500769:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1787763773.500783:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763773.500786:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763773.500806:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5ba000.
02000000:00000010:1.0:1787763773.500811:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca800.
00000100:00000001:2.0:1787763773.500813:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763773.500818:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763773.500827:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832658432/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787763773.500832:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763773.500851:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000100:00000001:2.0:1787763773.500851:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763773.500864:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763773.500868:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000040:2.0:1787763773.500870:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832658432/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763773.500883:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832658304, portal 25
00000100:00000001:2.0:1787763773.500887:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763773.500889:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00080000:2.0:1787763773.500892:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787763773.500893:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00080000:2.0:1787763773.500912:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1787763773.500915:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763773.500923:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:1.0:1787763773.500924:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832658304/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763798 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763773.500928:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763773.500938:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21488
00000400:00000200:2.0:1787763773.500963:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763773.500966:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763773.500975:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763773.500982:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000010:1.0:1787763773.500983:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763773.500992:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832658304, offset 0
00000100:00080000:2.0:1787763773.500994:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763773.501006:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:1.0:1787763773.501007:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787763773.501017:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763773.501021:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:1.0:1787763773.501031:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763773.501031:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763773.501042:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763773.501045:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787763773.501048:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763773.501065:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:2.0:1787763773.501071:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:1.0:1787763773.501072:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000020:00000001:2.0:1787763773.501080:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:1.0:1787763773.501084:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:2.0:1787763773.501089:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:1.0:1787763773.501095:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:2.0:1787763773.501095:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763773.501102:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:1.0:1787763773.501106:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1787763773.501107:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763773.501115:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:1.0:1787763773.501116:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763773.501136:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58378:58378:256:4294967295] 192.168.203.110@tcp LPNI seq info [58378:58378:8:4294967295]
00000400:00000200:1.0:1787763773.501143:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:2.0:1787763773.501165:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763773.501168:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763773.501169:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763773.501172:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763773.501174:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763773.501177:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787763773.501184:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:1.0:1787763773.501192:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000008:00000001:2.0:1787763773.501193:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763773.501201:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763773.501212:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000800:00000200:1.0:1787763773.501213:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000020:00000040:2.0:1787763773.501226:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000800:00000010:1.0:1787763773.501228:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
02000000:00000001:2.0:1787763773.501234:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763773.501237:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763773.501244:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:1.0:1787763773.501246:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:2.0:1787763773.501248:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763773.501255:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787763773.501256:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
02000000:00000010:2.0:1787763773.501264:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bbc00.
00000800:00000200:1.1:1787763773.501272:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787763773.501273:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787763773.501285:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000100:00100000:2.0:1787763773.501294:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832658560/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000100:1.0:1787763773.501304:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763773.501312:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000100:00000001:2.0:1787763773.501318:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763773.501321:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763773.501322:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:2.0:1787763773.501331:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832658560/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787763773.501335:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1787763773.501354:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763773.501362:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787763773.501363:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763773.501371:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000040:2.0:1787763773.501374:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:2.0:1787763773.501379:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787763773.501380:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:2.0:1787763773.501391:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787763773.501392:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763773.501393:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763773.501395:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763773.501400:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21488
00000100:00000200:1.0:1787763773.501408:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832658304/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763798 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787763773.501419:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763773.501436:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763773.501450:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763773.501457:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763773.501458:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787763773.501467:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763773.501474:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832658304/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763798 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787763773.501477:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763773.501491:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787763773.501510:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763773.501510:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:1.0:1787763773.501522:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000040:2.0:1787763773.501522:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787763773.501529:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:1.0:1787763773.501532:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1787763773.501534:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787763773.501539:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000200:1.0:1787763773.501550:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:2.0:1787763773.501552:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:1.0:1787763773.501558:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316161780
00000100:00000040:2.0:1787763773.501570:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000010:1.0:1787763773.501571:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4d80.
00000100:00000001:2.0:1787763773.501580:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787763773.501583:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000040:2.0:1787763773.501588:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763773.501595:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316161780
00000100:00000001:2.0:1787763773.501596:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000010:1.0:1787763773.501608:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4180.
00000020:00000040:2.0:1787763773.501609:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:1.0:1787763773.501623:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00080000:2.0:1787763773.501625:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763773.501634:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763773.501637:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763773.501638:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763773.501649:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763773.501651:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763773.501669:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763773.501670:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1787763773.501682:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787763773.501686:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00100000:1.0:1787763773.501690:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763773/real 1787763773]  req@ffff9751f8658780 x1874599832658304/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:2.0:1787763773.501693:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763773.501704:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763773.501718:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
00000020:00000040:2.0:1787763773.501730:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000400:00000001:1.0:1787763773.501731:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763773.501740:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:2.0:1787763773.501743:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:1.0:1787763773.501745:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
02000000:00000001:2.0:1787763773.501746:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763773.501763:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:1.0:1787763773.501766:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
02000000:00000001:2.0:1787763773.501766:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:1.0:1787763773.501780:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
02000000:00000001:2.0:1787763773.501780:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763773.501809:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5b8c00.
00000100:00000001:1.0:1787763773.501814:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:2.0:1787763773.501816:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787763773.501823:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832658304/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763773.501827:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832658688/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763773.501909:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763773.501915:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787763773.501916:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832658304/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787763773.501925:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832658688/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.501955:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763773.501966:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763773.501967:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763773.501972:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763773.501976:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763773.501980:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21498)
00000100:00100000:1.0:1787763773.501985:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832658304/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.501998:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763773.502006:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832658304/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.502016:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763773.502019:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.502021:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763773.502023:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.502026:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763773.502029:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763773.502033:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763773.502038:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763773.502042:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763773.502047:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832658304/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763773.502058:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832658304:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763773.502080:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.502083:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763773.502085:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832658304/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.502090:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763773.502092:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763773.502094:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763773.502097:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763773.502099:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763773.502101:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763773.502103:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763773.502104:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:1.0:1787763773.502107:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763773.502109:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763773.502110:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:1.0:1787763773.502113:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763773.502114:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.502116:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.502122:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763773.502123:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763773.502125:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763773.502127:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763773.502128:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763773.502131:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832658432/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.502137:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763773.502138:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763773.502140:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763773.502142:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763773.502144:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763773.502146:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832658432:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763773.502166:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763773.502169:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763773.502170:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763773.502172:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763773.502175:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763773.502177:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763773.502180:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763773.502182:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763773.502185:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763773.502188:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832658432, portal 10
00000100:00000001:1.0:1787763773.502190:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763773.502191:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:1.0:1787763773.502197:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832658432/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763798 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.502202:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763773.502204:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763773.502205:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763773.502207:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832658432, offset 0
00000400:00000200:1.0:1787763773.502210:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763773.502214:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763773.502217:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763773.502219:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763773.502222:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763773.502224:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763773.502226:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763773.502228:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58379:58379:256:4294967295] 192.168.203.110@tcp LPNI seq info [58379:58379:8:4294967295]
00000400:00000200:1.0:1787763773.502231:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763773.502237:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763773.502240:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763773.502243:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763773.502246:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763773.502248:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787763773.502253:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763773.502255:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:1.0:1787763773.502258:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763773.502260:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763773.502262:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763773.502265:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763773.502267:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763773.502268:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763773.502270:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763773.502273:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832658432/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763798 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.502279:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763773.502282:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832658432/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763798 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.502287:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763773.502288:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763773.502290:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763773.502293:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763773.502296:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763773.502298:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316160545
00000100:00000001:1.0:1787763773.502300:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316161780
00000100:00000001:1.0:1787763773.502302:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316160545
00000100:00000001:1.0:1787763773.502304:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316161780
00000100:00000001:1.0:1787763773.502305:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763773.502307:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763773.502309:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763773.502311:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763773/real 1787763773]  req@ffff9751f865ad00 x1874599832658432/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763773.502323:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763773.502343:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763773.502346:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763773.502348:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763773.502350:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763773.502353:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763773.502357:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832658432/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763773.502367:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832658432/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.502376:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763773.502377:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.502379:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763773.502381:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763773.502385:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832658432/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.502393:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763773.502398:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832658432/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.502406:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763773.502409:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.502411:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763773.502413:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.502415:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763773.502431:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763773.502434:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763773.502438:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763773.502441:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763773.502446:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832658432/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763773.502456:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832658432:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763773.502461:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763773.502466:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832658560/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.502474:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763773.502476:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763773.502478:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763773.502480:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763773.502483:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763773.502486:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832658560:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763773.502490:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763773.502493:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763773.502495:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763773.502497:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763773.502503:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763773.502505:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763773.502509:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763773.502512:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763773.502516:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763773.502520:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832658560, portal 10
00000100:00000001:1.0:1787763773.502522:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763773.502524:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787763773.502531:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832658560/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763798 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.502539:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763773.502542:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763773.502544:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763773.502547:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832658560, offset 0
00000400:00000200:1.0:1787763773.502553:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763773.502558:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763773.502562:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763773.502565:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763773.502569:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763773.502572:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763773.502575:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763773.502579:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58380:58380:256:4294967295] 192.168.203.110@tcp LPNI seq info [58380:58380:8:4294967295]
00000400:00000200:1.0:1787763773.502584:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763773.502592:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763773.502597:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763773.502601:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763773.502605:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763773.502609:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787763773.502614:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763773.502618:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:1.0:1787763773.502623:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763773.502625:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763773.502629:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763773.502633:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763773.502637:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763773.502639:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763773.502641:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763773.502646:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832658560/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763798 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.502655:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763773.502659:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832658560/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763798 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.502667:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763773.502669:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763773.502672:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763773.502676:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763773.502679:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763773.502682:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316160533
00000100:00000001:1.0:1787763773.502685:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316161780
00000100:00000001:1.0:1787763773.502688:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316160533
00000100:00000001:1.0:1787763773.502691:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316161780
00000100:00000001:1.0:1787763773.502693:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763773.502696:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763773.502698:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763773.502702:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763773/real 1787763773]  req@ffff9751f8659a40 x1874599832658560/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763773.502711:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763773.502713:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763773.502715:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763773.502717:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763773.502719:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763773.502722:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763773.502726:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832658560/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763773.502737:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832658560/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.502744:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763773.502746:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.502748:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763773.502749:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763773.502754:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832658560/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.502761:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763773.502766:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832658560/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.502774:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763773.502777:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.502779:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763773.502781:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.502783:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763773.502785:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763773.502788:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763773.502817:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763773.502820:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763773.502825:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832658560/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763773.502835:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832658560:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763773.502840:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763773.502845:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832658688/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.502854:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763773.502855:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763773.502858:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763773.502860:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763773.502863:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763773.502866:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832658688:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763773.502870:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763773.502872:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763773.502874:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763773.502877:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763773.502881:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c8c00.
02000000:00000001:1.0:1787763773.502883:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763773.502887:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763773.502890:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763773.502894:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763773.502898:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832658688, portal 4
00000100:00000001:1.0:1787763773.502901:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763773.502903:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787763773.502909:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832658688/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763798 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.502918:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763773.502920:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763773.502923:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763773.502926:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832658688, offset 0
00000400:00000200:1.0:1787763773.502930:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763773.502935:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763773.502939:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763773.502942:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763773.502945:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763773.502948:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763773.502951:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763773.502955:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58381:58381:256:4294967295] 192.168.203.110@tcp LPNI seq info [58381:58381:8:4294967295]
00000400:00000200:1.0:1787763773.502960:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763773.502968:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763773.502974:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763773.502977:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763773.502982:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763773.502985:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787763773.502990:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763773.502994:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:1.0:1787763773.502999:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763773.503001:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763773.503005:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763773.503009:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763773.503012:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763773.503015:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763773.503018:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763773.503022:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832658688/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763798 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.503040:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763773.503044:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832658688/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763798 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.503052:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763773.503054:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763773.503073:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763773.503076:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763773.503079:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763773.503080:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316160550
00000100:00000001:1.0:1787763773.503082:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316161780
00000100:00000001:1.0:1787763773.503084:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316160550
00000100:00000001:1.0:1787763773.503085:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316161780
00000100:00000001:1.0:1787763773.503087:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763773.503088:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763773.503089:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763773.503092:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763773/real 1787763773]  req@ffff9751f8658000 x1874599832658688/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763773.503097:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763773.503099:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763773.503100:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763773.503102:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763773.503103:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763773.503105:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763773.503107:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832658688/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763773.503113:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832658688/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.503117:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763773.503119:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.503120:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763773.503121:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763773.503124:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832658688/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.503129:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763773.503132:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832658688/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.503137:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763773.503138:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.503139:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763773.503140:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.503142:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763773.503143:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763773.503145:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763773.503147:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763773.503163:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763773.503167:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832658688/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763773.503173:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832658688:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763773.503175:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.503177:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763773.503179:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832658432/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.503184:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763773.503185:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763773.503187:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763773.503189:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763773.503191:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763773.503192:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763773.503194:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763773.503195:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:1.0:1787763773.503198:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763773.503199:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763773.503200:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:1.0:1787763773.503202:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763773.503203:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.503205:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763773.503207:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832658560/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.503212:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763773.503213:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763773.503214:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763773.503216:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763773.503216:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763773.503217:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763773.503219:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763773.503220:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:1.0:1787763773.503221:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763773.503222:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763773.503223:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787763773.503224:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763773.503225:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.503226:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763773.503228:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832658688/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763798 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763773.503233:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763773.503234:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763773.503235:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c8c00.
02000000:00000001:1.0:1787763773.503236:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763773.503237:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763773.503238:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763773.503239:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763773.503240:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:1.0:1787763773.503242:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763773.503243:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763773.503244:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:1.0:1787763773.503246:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763773.503247:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.503248:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763773.503251:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763773.503252:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763773.503253:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763773.503255:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763774.140238:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763774.140248:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763774.140253:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763774.140266:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763774.140280:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763774.140292:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763774.140307:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763774.140320:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763774.140334:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763774.140413:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787763774.140417:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787763774.140420:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:2.0:1787763774.780365:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316161780, at 4316161792
00000100:00000010:2.0:1787763774.780382:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4d80.
00000100:00000001:1.0:1787763775.164258:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763775.164265:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763775.164268:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763775.164272:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763775.164276:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763775.164279:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763775.164281:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763775.164283:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763775.164286:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763775.164434:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787763775.164440:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787763775.164444:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763776.188201:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763776.188210:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763776.188212:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763776.188218:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763776.188222:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763776.188225:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763776.188227:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763776.188254:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763776.188257:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763776.188466:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763776.188471:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763776.188475:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763777.212207:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763777.212217:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763777.212221:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763777.212230:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763777.212236:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763777.212243:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763777.212247:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763777.212251:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763777.212254:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763777.212408:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787763777.212415:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787763777.212421:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763778.236259:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763778.236266:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763778.236268:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763778.236273:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763778.236277:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763778.236279:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763778.236282:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763778.236284:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763778.236287:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763778.236468:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787763778.236474:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787763778.236480:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787763778.620222:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763778.620233:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787763778.620237:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763778.620241:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763778.620247:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763778.620256:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21493
00000400:00000200:2.0:1787763778.620284:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763778.620295:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763778.620301:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763778.620307:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763778.620313:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763778.620317:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763778.620322:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763778.620331:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763778.620334:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787763778.620340:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763778.620346:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787763778.620351:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787763778.620354:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787763778.620358:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763778.620361:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763778.620364:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763778.620366:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763778.620369:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787763778.620374:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763778.620377:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763778.620398:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763778.620404:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763778.620408:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763778.620411:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787763778.620416:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763778.620428:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000020:00000040:2.0:1787763778.620433:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787763778.620439:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763778.620442:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763778.620469:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763778.620472:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763778.620476:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763778.620484:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5ba000.
00000100:00000001:2.0:1787763778.620491:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763778.620500:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832658816/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763778.620510:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763778.620514:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763778.620519:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832658816/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763778.620537:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763778.620540:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763778.620542:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787763778.620544:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763778.620545:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763778.620547:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763778.620549:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21493
00000400:00000200:2.0:1787763778.620552:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763778.620554:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763778.620556:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763778.620558:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763778.620559:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763778.620561:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763778.620562:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763778.620564:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763778.620566:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:2.0:1787763778.620567:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763778.620569:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:2.0:1787763778.620571:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:2.0:1787763778.620573:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:2.0:1787763778.620593:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763778.620596:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763778.620599:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763778.620602:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763778.620613:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00000001:1.0:1787763778.620619:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:2.0:1787763778.620620:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763778.620630:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763778.620632:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
02000000:00000001:2.0:1787763778.620634:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787763778.620637:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
02000000:00000001:2.0:1787763778.620644:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763778.620652:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763778.620654:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832658816/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763778.620656:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787763778.620663:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1787763778.620668:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763778.620674:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763778.620691:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:2.0:1787763778.620692:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763778.620696:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763778.620697:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763778.620703:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763778.620709:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
00000100:00100000:1.0:1787763778.620710:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832658816:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763778.620725:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:2.0:1787763778.620725:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787763778.620735:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763778.620737:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763778.620744:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763778.620745:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763778.620772:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763778.620782:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c8c00.
00000100:00000001:2.0:1787763778.620782:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763778.620792:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763778.620793:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1787763778.620802:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:2.0:1787763778.620802:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763778.620819:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763778.620827:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
02000000:00000010:2.0:1787763778.620830:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5b9800.
00000100:00000200:1.0:1787763778.620835:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832658816, portal 25
00000100:00000001:2.0:1787763778.620837:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763778.620839:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763778.620845:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00100000:2.0:1787763778.620859:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832658944/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1787763778.620860:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832658816/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763778.620920:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763778.620921:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1787763778.620932:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763778.620936:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763778.620940:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763778.620943:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832658816, offset 0
00000100:00000040:2.0:1787763778.620948:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832658944/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787763778.620955:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787763778.620970:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763778.620971:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:2.0:1787763778.620978:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763778.620982:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:1.0:1787763778.620986:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:2.0:1787763778.620987:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763778.620996:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1787763778.620998:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1787763778.621004:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1787763778.621008:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00080000:2.0:1787763778.621011:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21493
00000400:00000200:1.0:1787763778.621015:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763778.621022:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763778.621022:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763778.621036:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:1.0:1787763778.621041:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58382:58382:256:4294967295] 192.168.203.110@tcp LPNI seq info [58382:58382:8:4294967295]
00000100:00080000:2.0:1787763778.621042:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763778.621054:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:1.0:1787763778.621055:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763778.621067:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763778.621074:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:1.0:1787763778.621086:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1787763778.621086:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763778.621093:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763778.621097:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000800:00000200:1.0:1787763778.621103:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000020:00000001:2.0:1787763778.621103:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763778.621110:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000010:1.0:1787763778.621112:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000020:00000001:2.0:1787763778.621117:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:2.0:1787763778.621120:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:1.0:1787763778.621127:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763778.621155:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000100:00000040:2.0:1787763778.621161:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:1.1:1787763778.621164:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787763778.621165:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.1:1787763778.621173:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000100:00000040:2.0:1787763778.621175:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763778.621186:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763778.621194:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000800:00000100:1.0:1787763778.621195:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763778.621204:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000100:00080000:2.0:1787763778.621210:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763778.621213:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763778.621216:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1787763778.621218:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763778.621223:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1787763778.621228:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:2.0:1787763778.621229:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763778.621233:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:1.0:1787763778.621236:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000008:00100000:2.0:1787763778.621237:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000010:1.0:1787763778.621240:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000008:00000001:2.0:1787763778.621244:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763778.621248:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763778.621249:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763778.621269:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832658816/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1787763778.621278:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787763778.621286:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000040:2.0:1787763778.621286:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000040:1.0:1787763778.621296:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832658816/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763803 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787763778.621296:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763778.621303:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763778.621313:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763778.621321:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1787763778.621324:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763778.621326:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763778.621329:0:257575:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000010:2.0:1787763778.621334:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bb800.
00000100:00000001:1.0:1787763778.621336:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763778.621344:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763778.621345:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763778.621359:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316166900
00000100:00100000:2.0:1787763778.621368:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832659072/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:1.0:1787763778.621371:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c46c0.
00000100:00000001:1.0:1787763778.621397:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787763778.621411:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763778.621412:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316166900
00000100:00000001:2.0:1787763778.621423:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763778.621424:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c47e0.
00000100:00000001:1.0:1787763778.621444:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000040:2.0:1787763778.621448:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832659072/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.621450:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763778.621466:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763778.621474:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763778.621484:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763778/real 1787763778]  req@ffff9751f86583c0 x1874599832658816/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763778.621485:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763778.621494:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1787763778.621507:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000001:1.0:1787763778.621508:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763778.621514:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00080000:2.0:1787763778.621515:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:1.0:1787763778.621519:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000100:00080000:2.0:1787763778.621522:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000200:1.0:1787763778.621525:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000100:00000001:2.0:1787763778.621525:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763778.621532:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:1.0:1787763778.621533:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763778.621540:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:2.0:1787763778.621541:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:1.0:1787763778.621550:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832658816/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787763778.621551:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21493
00000400:00000200:2.0:1787763778.621566:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000200:1.0:1787763778.621596:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832658816/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787763778.621604:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763778.621621:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787763778.621622:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763778.621629:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763778.621633:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763778.621636:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763778.621638:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787763778.621640:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763778.621647:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763778.621658:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00100000:1.0:1787763778.621659:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832658816/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787763778.621670:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763778.621674:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000100:00000001:1.0:1787763778.621678:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1787763778.621679:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763778.621684:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000040:1.0:1787763778.621690:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832658816/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787763778.621693:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:1.0:1787763778.621717:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000001:2.0:1787763778.621720:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:1.0:1787763778.621721:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763778.621732:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763778.621743:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763778.621743:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763778.621769:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763778.621771:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763778.621775:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763778.621782:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763778.621782:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787763778.621792:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763778.621811:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832658816/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787763778.621813:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763778.621823:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00100000:1.0:1787763778.621826:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832658816:192.168.203.110@tcp:250:kworker.0
00000020:00000040:2.0:1787763778.621832:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:1.0:1787763778.621833:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763778.621840:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00080000:2.0:1787763778.621847:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000040:1.0:1787763778.621848:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832658816/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763778.621858:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763778.621861:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763778.621865:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763778.621866:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763778.621870:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c8c00.
02000000:00000001:2.0:1787763778.621871:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763778.621880:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763778.621881:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763778.621888:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000100:00000001:2.0:1787763778.621888:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000020:00000040:1.0:1787763778.621898:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000008:00100000:2.0:1787763778.621900:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000020:00000001:1.0:1787763778.621902:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763778.621910:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
00000008:00000001:2.0:1787763778.621910:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763778.621917:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763778.621919:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763778.621923:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763778.621923:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000100:00000010:1.0:1787763778.621931:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000020:00000040:2.0:1787763778.621931:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:1.0:1787763778.621940:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
02000000:00000001:2.0:1787763778.621941:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1787763778.621949:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1787763778.621949:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763778.621965:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763778.621965:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763778.621973:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763778.621976:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763778.621978:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763778.621981:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763778.621981:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5ba000.
00000100:00000001:1.0:1787763778.621983:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763778.621988:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763778.621989:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763778.621992:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00100000:2.0:1787763778.621999:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832659200/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1787763778.622000:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832658944/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763778.622050:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763778.622051:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763778.622055:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763778.622055:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763778.622072:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763778.622075:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1787763778.622077:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832659200/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787763778.622081:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763778.622092:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832658944:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763778.622107:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1787763778.622111:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763778.622112:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763778.622121:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763778.622125:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21503)
02000000:00000001:1.0:1787763778.622126:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763778.622148:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c8c00.
02000000:00000001:1.0:1787763778.622152:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763778.622155:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763778.622157:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763778.622160:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763778.622162:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832658944, portal 10
00000100:00000001:1.0:1787763778.622163:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763778.622164:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787763778.622170:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832658944/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622175:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763778.622177:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763778.622178:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763778.622180:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832658944, offset 0
00000400:00000200:1.0:1787763778.622182:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763778.622185:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763778.622187:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763778.622190:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763778.622191:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763778.622193:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763778.622195:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763778.622197:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58383:58383:256:4294967295] 192.168.203.110@tcp LPNI seq info [58383:58383:8:4294967295]
00000400:00000200:1.0:1787763778.622200:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763778.622204:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763778.622207:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763778.622210:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763778.622213:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763778.622215:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787763778.622219:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763778.622220:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:1.0:1787763778.622223:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763778.622225:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763778.622227:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763778.622229:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763778.622231:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763778.622233:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763778.622234:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763778.622236:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832658944/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622242:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763778.622244:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832658944/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763803 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622253:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763778.622255:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763778.622257:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763778.622270:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763778.622272:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763778.622274:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316165664
00000100:00000001:1.0:1787763778.622276:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316166900
00000100:00000001:1.0:1787763778.622278:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316165664
00000100:00000001:1.0:1787763778.622279:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316166900
00000100:00000001:1.0:1787763778.622281:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763778.622282:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763778.622283:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763778.622286:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763778/real 1787763778]  req@ffff9751f8658000 x1874599832658944/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763778.622290:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763778.622291:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763778.622292:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763778.622293:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763778.622294:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763778.622296:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763778.622297:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832658944/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763778.622303:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832658944/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622307:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763778.622308:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763778.622309:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763778.622310:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763778.622312:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832658944/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622317:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763778.622320:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832658944/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622324:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763778.622326:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763778.622327:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763778.622328:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763778.622329:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763778.622340:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763778.622341:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763778.622344:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763778.622345:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763778.622347:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832658944/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763778.622352:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832658944:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763778.622355:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763778.622357:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832659072/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622362:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763778.622363:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763778.622365:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763778.622366:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763778.622367:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763778.622369:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832659072:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763778.622371:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763778.622372:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763778.622374:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763778.622375:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763778.622378:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763778.622379:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763778.622381:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763778.622382:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763778.622384:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763778.622386:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832659072, portal 10
00000100:00000001:1.0:1787763778.622388:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763778.622389:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787763778.622392:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832659072/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622408:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763778.622410:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763778.622412:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763778.622413:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832659072, offset 0
00000400:00000200:1.0:1787763778.622415:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763778.622417:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763778.622419:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763778.622421:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763778.622422:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763778.622424:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763778.622425:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763778.622427:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58384:58384:256:4294967295] 192.168.203.110@tcp LPNI seq info [58384:58384:8:4294967295]
00000400:00000200:1.0:1787763778.622430:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763778.622434:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763778.622437:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763778.622438:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763778.622441:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763778.622442:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787763778.622445:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763778.622446:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:1.0:1787763778.622449:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763778.622451:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763778.622452:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763778.622455:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763778.622456:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763778.622458:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763778.622459:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763778.622472:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832659072/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622477:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763778.622479:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832659072/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763803 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622484:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763778.622485:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763778.622486:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763778.622488:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763778.622490:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763778.622491:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316165652
00000100:00000001:1.0:1787763778.622493:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316166900
00000100:00000001:1.0:1787763778.622494:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316165652
00000100:00000001:1.0:1787763778.622496:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316166900
00000100:00000001:1.0:1787763778.622497:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763778.622499:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763778.622500:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763778.622502:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763778/real 1787763778]  req@ffff9751f8659a40 x1874599832659072/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763778.622506:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763778.622507:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763778.622508:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763778.622509:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763778.622510:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763778.622512:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763778.622514:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832659072/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763778.622520:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832659072/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622523:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763778.622524:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763778.622525:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763778.622526:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763778.622528:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832659072/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622532:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763778.622534:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832659072/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622538:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763778.622540:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763778.622540:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763778.622541:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763778.622542:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763778.622543:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763778.622545:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763778.622547:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763778.622548:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763778.622550:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832659072/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763778.622555:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832659072:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763778.622557:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763778.622559:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763778.622561:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832658944/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622565:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763778.622566:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763778.622567:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c8c00.
02000000:00000001:1.0:1787763778.622569:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763778.622570:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763778.622571:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763778.622572:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763778.622573:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:1.0:1787763778.622575:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763778.622576:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763778.622588:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:1.0:1787763778.622589:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763778.622590:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763778.622591:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763778.622593:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832659072/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622597:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763778.622598:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763778.622599:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763778.622601:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763778.622602:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763778.622603:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763778.622604:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763778.622605:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:1.0:1787763778.622606:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763778.622607:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763778.622608:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787763778.622609:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763778.622610:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763778.622611:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763778.622613:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763778.622614:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763778.622615:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763778.622617:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763778.622618:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763778.622621:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832659200/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622625:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763778.622626:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763778.622628:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763778.622629:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763778.622630:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763778.622632:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832659200:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763778.622634:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763778.622635:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763778.622636:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763778.622638:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763778.622639:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763778.622640:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763778.622643:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763778.622644:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763778.622646:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763778.622648:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832659200, portal 4
00000100:00000001:1.0:1787763778.622650:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763778.622651:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:1.0:1787763778.622654:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832659200/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622659:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763778.622660:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763778.622662:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763778.622663:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832659200, offset 0
00000400:00000200:1.0:1787763778.622665:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763778.622667:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763778.622669:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763778.622671:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763778.622681:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763778.622683:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763778.622685:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763778.622687:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58385:58385:256:4294967295] 192.168.203.110@tcp LPNI seq info [58385:58385:8:4294967295]
00000400:00000200:1.0:1787763778.622690:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763778.622694:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763778.622697:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763778.622699:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763778.622701:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763778.622703:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787763778.622706:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763778.622707:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:1.0:1787763778.622710:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763778.622711:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763778.622713:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763778.622715:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763778.622717:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763778.622718:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763778.622719:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763778.622722:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832659200/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622727:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763778.622729:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832659200/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763803 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622733:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763778.622734:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763778.622735:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763778.622737:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763778.622739:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763778.622740:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316165669
00000100:00000001:1.0:1787763778.622741:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316166900
00000100:00000001:1.0:1787763778.622743:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316165669
00000100:00000001:1.0:1787763778.622746:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316166900
00000100:00000001:1.0:1787763778.622747:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763778.622749:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763778.622750:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763778.622752:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763778/real 1787763778]  req@ffff9751f865ad00 x1874599832659200/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763778.622769:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763778.622770:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763778.622771:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763778.622772:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763778.622773:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763778.622775:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763778.622777:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832659200/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763778.622782:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832659200/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622787:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763778.622788:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763778.622789:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763778.622790:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763778.622792:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832659200/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622796:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763778.622798:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832659200/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622803:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763778.622805:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763778.622805:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763778.622806:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763778.622819:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763778.622820:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763778.622822:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763778.622825:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763778.622826:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763778.622829:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832659200/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763778.622833:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832659200:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763778.622835:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763778.622836:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763778.622838:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832659200/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763803 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763778.622842:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763778.622843:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763778.622844:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763778.622846:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763778.622847:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763778.622848:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763778.622849:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763778.622850:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:1.0:1787763778.622852:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763778.622853:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763778.622854:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:1.0:1787763778.622856:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763778.622856:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763778.622857:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763778.622859:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763778.622860:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763778.622861:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763778.622862:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763779.260296:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763779.260310:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763779.260318:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763779.260327:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763779.260338:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763779.260348:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763779.260356:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763779.260362:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763779.260369:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763779.260412:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763779.260419:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763779.260423:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000040:2.0:1787763779.900254:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316166900, at 4316166912
00000100:00000010:2.0:1787763779.900263:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c46c0.
00000100:00000001:1.0:1787763780.284266:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763780.284275:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763780.284278:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763780.284285:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763780.284290:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763780.284313:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763780.284317:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763780.284321:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763780.284325:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763780.284361:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787763780.284366:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787763780.284369:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763781.308214:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763781.308226:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763781.308229:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763781.308236:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763781.308242:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763781.308246:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763781.308251:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763781.308254:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763781.308259:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763781.308463:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787763781.308472:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787763781.308478:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763782.332272:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763782.332289:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763782.332293:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763782.332312:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763782.332323:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763782.332334:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763782.332339:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763782.332341:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763782.332345:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763782.332397:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763782.332406:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763782.332412:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763783.356272:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763783.356285:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763783.356289:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763783.356297:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763783.356305:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763783.356311:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763783.356314:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763783.356316:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763783.356320:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763783.356322:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763783.356325:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763783.356326:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763783.356332:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763783.356335:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763783.356471:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787763783.356478:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787763783.356483:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763783.740263:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763783.740275:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763783.740281:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763783.740287:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:3.0:1787763783.740290:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:0.0:1787763783.740306:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763783.740311:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763783.740318:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763783.740327:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787763783.740336:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1787763783.740343:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21498
00000100:00000001:3.0:1787763783.740348:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763783.740357:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000400:00000200:0.0:1787763783.740364:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763783.740366:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763783.740376:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763783.740381:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763783.740390:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763783.740396:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763783.740401:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763783.740407:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763783.740411:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763783.740416:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763783.740419:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763783.740422:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763783.740427:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763783.740430:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763783.740433:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763783.740438:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763783.740440:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763783.740444:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763783.740446:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763783.740450:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763783.740455:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763783.740459:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763783.740463:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763783.740468:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763783.740472:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763783.740475:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763783.740478:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763783.740489:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
00000020:00000040:0.0:1787763783.740493:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763783.740498:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763783.740500:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763783.740504:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763783.740508:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763783.740513:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763783.740523:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763783.740531:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763783.740544:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832659328/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763783.740557:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763783.740563:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763783.740570:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832659328/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763783.740600:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763783.740605:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763783.740610:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763783.740613:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763783.740616:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763783.740621:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763783.740625:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21498
00000400:00000200:0.0:1787763783.740632:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763783.740638:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763783.740641:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787763783.740645:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1787763783.740647:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763783.740654:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787763783.740655:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:0.0:1787763783.740659:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763783.740662:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1787763783.740665:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763783.740671:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763783.740673:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763783.740678:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:1.0:1787763783.740680:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f840 x1874599832659328/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787763783.740681:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763783.740690:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763783.740697:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763783.740710:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763783.740717:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787763783.740746:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763783.740750:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763783.740755:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:1.0:1787763783.740758:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1787763783.740761:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:1.0:1787763783.740767:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:0.0:1787763783.740768:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763783.740780:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:1.0:1787763783.740780:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763783.740790:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763783.740796:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832659328:192.168.203.110@tcp:250:kworker.0
02000000:00000001:0.0:1787763783.740797:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787763783.740806:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763783.740807:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763783.740814:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763783.740815:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763783.740821:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763783.740822:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1787763783.740829:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:0.0:1787763783.740831:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763783.740837:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763783.740840:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca400.
00000100:00000001:0.0:1787763783.740841:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763783.740852:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763783.740854:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787763783.740861:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000200:1.0:1787763783.740866:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787763783.740869:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763783.740874:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763783.740876:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000100:00000001:0.0:1787763783.740878:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763783.740885:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763783.740889:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763783.740891:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
02000000:00000010:0.0:1787763783.740898:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000200:1.0:1787763783.740903:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832659328, portal 25
00000100:00000001:0.0:1787763783.740907:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763783.740911:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763783.740916:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049532480 : -115096660019136 : ffff9751f876f840)
00000100:00100000:0.0:1787763783.740924:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832659456/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1787763783.740933:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f840 x1874599832659328/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763808 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763783.740955:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763783.740963:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763783.740965:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763783.740975:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832659456/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787763783.740977:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763783.740987:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763783.740999:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832659328, offset 0
00000100:00000001:0.0:1787763783.741001:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763783.741011:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763783.741020:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:1.0:1787763783.741021:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763783.741027:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763783.741032:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763783.741036:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1787763783.741041:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:0.0:1787763783.741047:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21498
00000400:00000200:1.0:1787763783.741057:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763783.741065:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763783.741072:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763783.741077:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763783.741081:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763783.741086:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58386:58386:256:4294967295] 192.168.203.110@tcp LPNI seq info [58386:58386:8:4294967295]
00000400:00000200:1.0:1787763783.741092:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763783.741101:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763783.741118:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:1.0:1787763783.741119:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00080000:0.0:1787763783.741125:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763783.741167:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763783.741171:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763783.741175:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763783.741178:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763783.741187:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:1.0:1787763783.741188:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000020:00000040:0.0:1787763783.741191:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763783.741197:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000010:1.0:1787763783.741200:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000020:00000040:0.0:1787763783.741201:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763783.741211:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763783.741214:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:1.0:1787763783.741222:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763783.741227:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763783.741234:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.0:1787763783.741234:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000100:00000040:0.0:1787763783.741244:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763783.741250:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:1.1:1787763783.741256:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:0.0:1787763783.741259:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000800:00000200:1.1:1787763783.741266:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000100:00080000:0.0:1787763783.741275:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000800:00000100:1.0:1787763783.741282:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763783.741287:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1787763783.741289:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
02000000:00000001:0.0:1787763783.741299:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787763783.741303:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:0.0:1787763783.741309:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1787763783.741311:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:0.0:1787763783.741322:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763783.741325:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000100:00000001:0.0:1787763783.741333:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:1.0:1787763783.741333:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000008:00100000:0.0:1787763783.741342:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1787763783.741344:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000008:00000001:0.0:1787763783.741357:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787763783.741358:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f840 x1874599832659328/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763808 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763783.741369:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763783.741382:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000100:00000001:1.0:1787763783.741385:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000040:0.0:1787763783.741388:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000040:1.0:1787763783.741396:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f840 x1874599832659328/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763808 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763783.741398:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763783.741403:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763783.741411:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763783.741416:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763783.741422:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763783.741427:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763783.741431:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:1.0:1787763783.741432:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763783.741443:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763783.741446:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763783.741456:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832659584/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787763783.741461:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763783.741471:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316172020
00000100:00000001:0.0:1787763783.741481:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763783.741487:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763783.741488:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4960.
00000100:00000040:0.0:1787763783.741501:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832659584/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.741504:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763783.741519:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763783.741523:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763783.741527:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763783.741532:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787763783.741534:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316172020
00000100:00080000:0.0:1787763783.741540:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000010:1.0:1787763783.741544:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4de0.
00000100:00000001:0.0:1787763783.741546:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763783.741551:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763783.741556:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1787763783.741561:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00080000:0.0:1787763783.741562:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21498
00000100:00000001:1.0:1787763783.741571:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1787763783.741578:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763783.741581:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763783.741589:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1787763783.741590:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00080000:0.0:1787763783.741596:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763783.741607:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00100000:1.0:1787763783.741608:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763783/real 1787763783]  req@ffff9751f876f840 x1874599832659328/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763783.741611:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763783.741619:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763783.741628:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763783.741636:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763783.741641:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763783.741648:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000001:1.0:1787763783.741650:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000040:0.0:1787763783.741652:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000400:00000001:1.0:1787763783.741656:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000020:00000001:0.0:1787763783.741658:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000010:1.0:1787763783.741663:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000020:00000001:0.0:1787763783.741665:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000200:1.0:1787763783.741674:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000100:00000040:0.0:1787763783.741675:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000010:1.0:1787763783.741683:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:0.0:1787763783.741684:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763783.741693:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763783.741694:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:0.0:1787763783.741700:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000200:1.0:1787763783.741708:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f840 x1874599832659328/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787763783.741709:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763783.741740:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763783.741753:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763783.741761:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763783.741768:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:1.0:1787763783.741771:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f840 x1874599832659328/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763783.741773:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763783.741782:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763783.741787:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1787763783.741794:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000008:00000001:0.0:1787763783.741796:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763783.741800:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763783.741802:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763783.741811:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000010:0.0:1787763783.741817:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000100:00000001:1.0:1787763783.741818:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:0.0:1787763783.741829:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00100000:1.0:1787763783.741832:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f840 x1874599832659328/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763783.741835:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763783.741840:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763783.741847:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763783.741852:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763783.741856:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763783.741861:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000010:0.0:1787763783.741866:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000040:1.0:1787763783.741874:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f840 x1874599832659328/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763783.741876:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763783.741894:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832659712/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763783.741904:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763783.741907:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763783.741912:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832659712/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.741925:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763783.741931:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763783.741934:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763783.741940:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:0.0:1787763783.741941:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21508)
00000100:00000001:1.0:1787763783.741943:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763783.741952:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763783.741958:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763783.741963:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763783.741970:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763783.741975:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763783.741983:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f840 x1874599832659328/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763783.741995:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832659328:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763783.742001:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763783.742004:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763783.742008:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f840 x1874599832659328/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.742017:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763783.742021:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763783.742024:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763783.742030:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763783.742034:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763783.742037:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763783.742041:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763783.742044:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015c00.
02000000:00000001:1.0:1787763783.742050:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763783.742052:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763783.742054:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f840.
00000100:00000001:1.0:1787763783.742058:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763783.742060:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763783.742063:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763783.742071:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763783.742073:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763783.742076:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763783.742079:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763783.742081:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763783.742086:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e940 x1874599832659456/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.742095:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763783.742097:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763783.742100:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763783.742103:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763783.742106:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763783.742109:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832659456:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763783.742113:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763783.742116:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763783.742119:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763783.742121:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763783.742125:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763783.742128:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763783.742158:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763783.742163:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763783.742168:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763783.742172:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832659456, portal 10
00000100:00000001:1.0:1787763783.742175:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763783.742177:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049528640 : -115096660022976 : ffff9751f876e940)
00000100:00000040:1.0:1787763783.742186:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e940 x1874599832659456/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763808 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.742195:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763783.742198:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763783.742201:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763783.742204:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832659456, offset 0
00000400:00000200:1.0:1787763783.742209:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763783.742214:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763783.742218:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763783.742222:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763783.742226:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763783.742230:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763783.742233:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763783.742237:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58387:58387:256:4294967295] 192.168.203.110@tcp LPNI seq info [58387:58387:8:4294967295]
00000400:00000200:1.0:1787763783.742242:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763783.742251:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763783.742256:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763783.742260:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763783.742265:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763783.742269:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787763783.742275:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763783.742279:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1787763783.742283:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763783.742286:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763783.742290:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763783.742294:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763783.742298:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763783.742301:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763783.742304:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763783.742308:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e940 x1874599832659456/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763808 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.742318:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763783.742322:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e940 x1874599832659456/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763808 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.742330:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763783.742332:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763783.742337:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763783.742342:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763783.742345:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763783.742348:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316170785
00000100:00000001:1.0:1787763783.742351:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316172020
00000100:00000001:1.0:1787763783.742354:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316170785
00000100:00000001:1.0:1787763783.742360:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316172020
00000100:00000001:1.0:1787763783.742363:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763783.742365:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763783.742368:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763783.742373:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763783/real 1787763783]  req@ffff9751f876e940 x1874599832659456/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763783.742382:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763783.742383:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763783.742385:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763783.742388:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763783.742390:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763783.742392:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763783.742396:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e940 x1874599832659456/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763783.742406:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e940 x1874599832659456/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.742415:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763783.742416:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763783.742418:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763783.742420:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763783.742424:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e940 x1874599832659456/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.742432:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763783.742437:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e940 x1874599832659456/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.742445:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763783.742448:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763783.742450:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763783.742451:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763783.742455:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763783.742468:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763783.742472:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763783.742476:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763783.742479:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763783.742484:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e940 x1874599832659456/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763783.742493:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832659456:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763783.742498:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763783.742502:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cf00 x1874599832659584/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.742510:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763783.742512:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763783.742514:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763783.742516:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763783.742519:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763783.742523:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832659584:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763783.742527:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763783.742529:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763783.742532:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763783.742534:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763783.742538:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c8c00.
02000000:00000001:1.0:1787763783.742541:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763783.742545:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763783.742547:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763783.742551:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763783.742555:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832659584, portal 10
00000100:00000001:1.0:1787763783.742557:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763783.742559:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049521920 : -115096660029696 : ffff9751f876cf00)
00000100:00000040:1.0:1787763783.742567:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cf00 x1874599832659584/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763808 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.742575:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763783.742578:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763783.742581:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763783.742584:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832659584, offset 0
00000400:00000200:1.0:1787763783.742588:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763783.742592:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763783.742596:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763783.742599:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763783.742602:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763783.742605:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763783.742608:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763783.742612:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58388:58388:256:4294967295] 192.168.203.110@tcp LPNI seq info [58388:58388:8:4294967295]
00000400:00000200:1.0:1787763783.742617:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763783.742625:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763783.742631:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763783.742635:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763783.742639:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763783.742643:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787763783.742647:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763783.742650:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1787763783.742656:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763783.742658:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763783.742662:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763783.742666:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763783.742669:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763783.742672:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763783.742675:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763783.742679:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cf00 x1874599832659584/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763808 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.742688:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763783.742692:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cf00 x1874599832659584/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763808 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.742709:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763783.742711:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763783.742714:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763783.742717:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763783.742741:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763783.742745:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316170773
00000100:00000001:1.0:1787763783.742748:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316172020
00000100:00000001:1.0:1787763783.742751:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316170773
00000100:00000001:1.0:1787763783.742753:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316172020
00000100:00000001:1.0:1787763783.742756:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763783.742758:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763783.742761:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763783.742765:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763783/real 1787763783]  req@ffff9751f876cf00 x1874599832659584/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763783.742775:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763783.742777:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763783.742779:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763783.742782:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763783.742784:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763783.742786:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763783.742790:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cf00 x1874599832659584/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763783.742801:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cf00 x1874599832659584/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.742809:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763783.742810:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763783.742812:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763783.742814:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763783.742818:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cf00 x1874599832659584/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.742827:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763783.742832:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cf00 x1874599832659584/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.742841:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763783.742843:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763783.742845:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763783.742847:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763783.742849:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763783.742851:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763783.742854:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763783.742858:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763783.742860:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763783.742865:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cf00 x1874599832659584/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763783.742874:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832659584:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763783.742878:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763783.742882:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f0c0 x1874599832659712/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.742890:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763783.742892:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763783.742894:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763783.742897:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763783.742900:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763783.742903:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832659712:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763783.742907:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763783.742909:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763783.742912:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763783.742914:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763783.742919:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763783.742921:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763783.742925:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763783.742928:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763783.742932:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763783.742936:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832659712, portal 4
00000100:00000001:1.0:1787763783.742938:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763783.742940:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049530560 : -115096660021056 : ffff9751f876f0c0)
00000100:00000040:1.0:1787763783.742947:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f0c0 x1874599832659712/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763808 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.742955:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763783.742958:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763783.742960:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763783.742963:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832659712, offset 0
00000400:00000200:1.0:1787763783.742968:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763783.742971:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763783.742975:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763783.742979:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763783.742982:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763783.742985:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763783.742988:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763783.742992:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58389:58389:256:4294967295] 192.168.203.110@tcp LPNI seq info [58389:58389:8:4294967295]
00000400:00000200:1.0:1787763783.742997:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763783.743005:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763783.743010:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763783.743014:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763783.743018:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763783.743021:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787763783.743026:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763783.743030:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1787763783.743034:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763783.743037:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763783.743041:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763783.743045:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763783.743048:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763783.743051:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763783.743053:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763783.743058:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f0c0 x1874599832659712/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763808 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.743067:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763783.743071:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f0c0 x1874599832659712/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763808 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.743079:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763783.743081:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763783.743085:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763783.743088:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763783.743092:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763783.743094:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316170790
00000100:00000001:1.0:1787763783.743097:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316172020
00000100:00000001:1.0:1787763783.743100:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316170790
00000100:00000001:1.0:1787763783.743103:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316172020
00000100:00000001:1.0:1787763783.743105:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763783.743107:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763783.743110:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763783.743114:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763783/real 1787763783]  req@ffff9751f876f0c0 x1874599832659712/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763783.743122:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763783.743124:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763783.743126:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763783.743128:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763783.743130:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763783.743159:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763783.743164:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f0c0 x1874599832659712/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763783.743175:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f0c0 x1874599832659712/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.743183:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763783.743185:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763783.743187:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763783.743189:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763783.743193:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f0c0 x1874599832659712/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.743201:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763783.743206:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f0c0 x1874599832659712/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.743214:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763783.743218:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763783.743220:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763783.743221:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763783.743223:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763783.743225:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763783.743228:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763783.743232:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763783.743234:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763783.743239:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f0c0 x1874599832659712/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763783.743248:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832659712:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763783.743252:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763783.743255:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763783.743259:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e940 x1874599832659456/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.743268:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763783.743270:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763783.743273:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763783.743275:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763783.743277:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763783.743279:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763783.743282:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763783.743284:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:1.0:1787763783.743287:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763783.743289:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763783.743291:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e940.
00000100:00000001:1.0:1787763783.743296:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763783.743298:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763783.743300:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763783.743304:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cf00 x1874599832659584/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.743312:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763783.743314:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763783.743316:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c8c00.
02000000:00000001:1.0:1787763783.743318:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763783.743320:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763783.743321:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763783.743323:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763783.743325:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016800.
02000000:00000001:1.0:1787763783.743328:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763783.743330:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763783.743331:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cf00.
00000100:00000001:1.0:1787763783.743335:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763783.743337:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763783.743339:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763783.743342:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f0c0 x1874599832659712/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763808 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763783.743349:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763783.743352:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763783.743354:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763783.743356:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763783.743358:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763783.743360:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763783.743362:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763783.743364:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014800.
02000000:00000001:1.0:1787763783.743366:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763783.743367:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763783.743369:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f0c0.
00000100:00000001:1.0:1787763783.743371:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763783.743373:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763783.743375:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763783.743379:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763783.743380:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763783.743383:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763783.743385:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763784.380265:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763784.380280:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763784.380286:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763784.380287:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763784.380297:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763784.380299:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763784.380305:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763784.380310:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763784.380311:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763784.380322:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763784.380323:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763784.380327:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763784.380334:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763784.380361:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763784.380375:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763784.380441:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787763784.380448:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787763784.380453:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763784.764296:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763784.764308:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763784.764312:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763784.764321:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763784.764328:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763784.764333:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763784.764338:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763784.764344:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763784.764348:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763785.020282:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316172020, at 4316172032
00000100:00000010:3.0:1787763785.020303:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4de0.
00000100:00000040:2.0:1787763785.020352:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316172020, at 4316172032
00000100:00000010:2.0:1787763785.020376:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4960.
00010000:00000001:3.0:1787763785.404227:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763785.404235:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763785.404243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763785.404249:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763785.404253:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763785.404258:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763785.404264:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:1.0:1787763785.404266:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763785.404269:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763785.404278:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763785.404278:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763785.404285:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763785.404287:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763785.404294:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763785.404297:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763785.404303:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763785.404304:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763785.404311:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763785.404317:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763785.404320:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763785.404323:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763785.404377:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763785.404383:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763785.404387:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763785.788254:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763785.788266:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763785.788272:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763785.788281:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763785.788290:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763785.788297:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763785.788302:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763785.788308:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763785.788315:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763786.428172:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763786.428179:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763786.428182:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763786.428187:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763786.428191:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763786.428192:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763786.428194:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763786.428197:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763786.428198:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763786.428203:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763786.428205:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763786.428206:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763786.428208:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763786.428215:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763786.428217:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763786.428259:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763786.428262:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763786.428265:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763786.812188:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763786.812195:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763786.812197:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763786.812201:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763786.812204:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763786.812207:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763786.812209:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763786.812211:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763786.812213:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763787.452270:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763787.452273:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763787.452282:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763787.452287:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763787.452294:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763787.452295:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763787.452300:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763787.452308:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763787.452311:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763787.452319:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763787.452320:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763787.452330:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763787.452337:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763787.452342:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763787.452348:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763787.452381:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787763787.452387:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787763787.452391:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763787.836236:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763787.836248:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763787.836250:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763787.836258:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763787.836265:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763787.836271:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763787.836274:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763787.836277:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763787.836281:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763788.476222:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763788.476230:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763788.476234:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763788.476240:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763788.476245:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763788.476250:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.476254:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763788.476257:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.476262:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763788.476284:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763788.476293:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763788.476301:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763788.476304:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763788.476311:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763788.476313:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763788.476420:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787763788.476427:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787763788.476431:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763788.860281:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:0.0:1787763788.860284:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787763788.860297:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1787763788.860303:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787763788.860305:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763788.860311:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:3.0:1787763788.860317:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:0.0:1787763788.860319:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763788.860328:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763788.860331:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787763788.860336:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763788.860343:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:0.0:1787763788.860346:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21503
00000100:00000001:3.0:1787763788.860349:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763788.860358:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763788.860366:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763788.860377:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763788.860382:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763788.860387:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763788.860394:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763788.860398:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763788.860402:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763788.860409:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763788.860412:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763788.860416:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763788.860422:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763788.860425:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763788.860429:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763788.860433:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763788.860435:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763788.860437:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763788.860440:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763788.860444:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763788.860448:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763788.860451:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763788.860456:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763788.860460:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763788.860464:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763788.860467:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763788.860470:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763788.860481:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763788.860486:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763788.860491:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763788.860494:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763788.860498:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763788.860502:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763788.860509:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763788.860536:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787763788.860553:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763788.860567:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832659840/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763788.860592:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763788.860598:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763788.860605:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832659840/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763788.860637:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763788.860641:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763788.860646:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763788.860648:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763788.860651:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763788.860655:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763788.860659:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21503
00000400:00000200:0.0:1787763788.860666:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763788.860674:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763788.860677:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763788.860681:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763788.860703:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763788.860710:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787763788.860714:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:0.0:1787763788.860720:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763788.860724:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1787763788.860725:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763788.860733:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763788.860735:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763788.860738:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763788.860741:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763788.860747:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787763788.860749:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832659840/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787763788.860751:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763788.860763:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763788.860771:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763788.860776:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763788.860789:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763788.860803:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00000001:1.0:1787763788.860809:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:0.0:1787763788.860811:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763788.860817:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763788.860818:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763788.860828:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763788.860832:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763788.860838:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763788.860839:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763788.860845:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763788.860850:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1787763788.860850:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1787763788.860860:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00100000:1.0:1787763788.860866:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832659840:192.168.203.110@tcp:250:kworker.0
00000008:00000001:0.0:1787763788.860867:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763788.860876:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763788.860889:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000100:00000001:1.0:1787763788.860889:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:0.0:1787763788.860898:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763788.860904:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763788.860905:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763788.860912:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763788.860915:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763788.860916:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763788.860919:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763788.860926:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763788.860927:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787763788.860934:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
02000000:00000010:1.0:1787763788.860941:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca800.
00000100:00000001:0.0:1787763788.860942:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763788.860954:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763788.860961:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832659968/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787763788.860976:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763788.860989:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:1.0:1787763788.860990:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000100:00000001:0.0:1787763788.860995:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763788.861006:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000040:0.0:1787763788.861007:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832659968/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763788.861026:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832659840, portal 25
00000100:00000001:0.0:1787763788.861030:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763788.861036:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00080000:0.0:1787763788.861037:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787763788.861042:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00080000:0.0:1787763788.861044:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763788.861047:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763788.861053:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763788.861059:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:1.0:1787763788.861063:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832659840/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763813 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787763788.861065:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21503
00000400:00000200:0.0:1787763788.861085:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763788.861097:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763788.861104:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787763788.861109:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763788.861114:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763788.861120:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763788.861125:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787763788.861129:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763788.861164:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763788.861169:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763788.861171:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763788.861174:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:1.0:1787763788.861175:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000020:00000040:0.0:1787763788.861177:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763788.861185:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000200:1.0:1787763788.861186:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832659840, offset 0
00000020:00000001:0.0:1787763788.861193:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763788.861213:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:1.0:1787763788.861214:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763788.861228:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763788.861238:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:1.0:1787763788.861239:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763788.861251:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763788.861259:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:1.0:1787763788.861261:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00080000:0.0:1787763788.861276:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:1.0:1787763788.861280:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1787763788.861286:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763788.861297:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787763788.861297:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
02000000:00000001:0.0:1787763788.861307:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1787763788.861308:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
02000000:00000001:0.0:1787763788.861315:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763788.861321:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:1.0:1787763788.861322:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000008:00100000:0.0:1787763788.861325:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763788.861335:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763788.861338:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58390:58390:256:4294967295] 192.168.203.110@tcp LPNI seq info [58390:58390:8:4294967295]
00000100:00000001:0.0:1787763788.861340:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763788.861349:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763788.861356:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000400:00000200:1.0:1787763788.861357:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787763788.861368:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763788.861373:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763788.861377:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763788.861382:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763788.861389:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763788.861397:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
02000000:00000010:0.0:1787763788.861401:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787763788.861412:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787763788.861416:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00100000:0.0:1787763788.861426:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832660096/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000010:1.0:1787763788.861430:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000100:00000001:0.0:1787763788.861450:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763788.861457:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787763788.861457:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763788.861471:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832660096/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.0:1787763788.861474:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:0.0:1787763788.861502:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787763788.861506:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763788.861508:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763788.861521:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:1.1:1787763788.861524:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000100:00080000:0.0:1787763788.861527:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763788.861541:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763788.861544:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763788.861553:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000100:1.0:1787763788.861554:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763788.861559:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000800:00000010:1.0:1787763788.861563:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000100:00080000:0.0:1787763788.861567:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21503
00000400:00000200:1.0:1787763788.861577:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787763788.861578:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763788.861590:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763788.861599:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1787763788.861601:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:0.0:1787763788.861606:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763788.861610:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763788.861613:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763788.861619:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763788.861625:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763788.861629:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763788.861632:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:1.0:1787763788.861633:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000020:00000040:0.0:1787763788.861636:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000400:00000010:1.0:1787763788.861643:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000020:00000001:0.0:1787763788.861644:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763788.861655:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:1.0:1787763788.861660:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:0.0:1787763788.861661:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763788.861669:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763788.861681:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000200:1.0:1787763788.861683:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e580 x1874599832659840/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763813 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763788.861711:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787763788.861714:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000040:0.0:1787763788.861718:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763788.861727:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000040:1.0:1787763788.861727:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832659840/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763813 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763788.861737:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763788.861745:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763788.861751:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763788.861762:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763788.861766:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763788.861767:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763788.861772:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1787763788.861773:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000008:00000001:0.0:1787763788.861781:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763788.861785:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763788.861789:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763788.861791:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763788.861800:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763788.861809:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:1.0:1787763788.861810:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787763788.861813:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763788.861819:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1787763788.861819:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316177140
02000000:00000001:0.0:1787763788.861823:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763788.861827:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763788.861834:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000010:1.0:1787763788.861835:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4960.
00000100:00000001:0.0:1787763788.861842:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763788.861849:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00100000:0.0:1787763788.861852:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832660224/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763788.861862:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316177140
00000100:00000001:0.0:1787763788.861864:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763788.861870:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763788.861876:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4de0.
00000100:00000040:0.0:1787763788.861881:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832660224/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.861885:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763788.861898:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763788.861909:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763788.861912:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763788.861918:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21513)
00000100:00000001:1.0:1787763788.861918:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763788.861934:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763788/real 1787763788]  req@ffff9751f876e580 x1874599832659840/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763788.861950:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763788.861953:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763788.861956:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763788.861960:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763788.861962:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763788.861965:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763788.861969:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832659840/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763788.861979:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832659840/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.861988:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763788.861990:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.861992:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763788.861994:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763788.861999:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832659840/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.862006:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763788.862011:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832659840/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.862019:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763788.862022:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.862024:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763788.862025:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.862028:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763788.862030:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763788.862034:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763788.862039:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763788.862042:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763788.862047:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832659840/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763788.862057:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832659840:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763788.862063:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.862067:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763788.862070:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832659840/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.862079:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763788.862082:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763788.862085:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763788.862089:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763788.862092:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763788.862094:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763788.862096:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763788.862099:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014400.
02000000:00000001:1.0:1787763788.862113:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763788.862116:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763788.862118:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:1.0:1787763788.862123:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763788.862125:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.862127:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.862162:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763788.862164:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763788.862167:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763788.862169:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763788.862171:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763788.862176:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832659968/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.862184:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763788.862186:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763788.862188:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763788.862190:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763788.862193:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763788.862197:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832659968:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763788.862200:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763788.862203:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763788.862205:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763788.862207:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763788.862210:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763788.862212:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763788.862217:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763788.862220:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763788.862223:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763788.862227:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832659968, portal 10
00000100:00000001:1.0:1787763788.862229:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763788.862230:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
00000100:00000040:1.0:1787763788.862238:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832659968/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763813 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.862246:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763788.862249:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763788.862251:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763788.862254:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832659968, offset 0
00000400:00000200:1.0:1787763788.862259:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763788.862264:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763788.862269:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763788.862272:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763788.862276:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763788.862279:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763788.862282:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763788.862286:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58391:58391:256:4294967295] 192.168.203.110@tcp LPNI seq info [58391:58391:8:4294967295]
00000400:00000200:1.0:1787763788.862291:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763788.862298:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763788.862304:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763788.862308:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763788.862312:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763788.862316:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787763788.862321:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763788.862324:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1787763788.862328:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763788.862331:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763788.862334:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763788.862338:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763788.862341:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763788.862343:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763788.862346:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763788.862350:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876ed00 x1874599832659968/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763813 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.862359:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763788.862363:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832659968/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763813 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.862370:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763788.862371:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763788.862375:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763788.862379:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763788.862382:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763788.862384:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316175905
00000100:00000001:1.0:1787763788.862387:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316177140
00000100:00000001:1.0:1787763788.862390:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316175905
00000100:00000001:1.0:1787763788.862392:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316177140
00000100:00000001:1.0:1787763788.862394:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763788.862396:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763788.862399:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763788.862403:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763788/real 1787763788]  req@ffff9751f876ed00 x1874599832659968/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763788.862413:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763788.862414:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763788.862416:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763788.862418:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763788.862420:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763788.862422:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763788.862426:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832659968/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763788.862435:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832659968/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.862443:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763788.862444:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.862447:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763788.862448:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763788.862453:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832659968/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.862461:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763788.862466:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832659968/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.862473:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763788.862476:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.862478:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763788.862480:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.862482:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763788.862498:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763788.862502:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763788.862507:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763788.862510:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763788.862515:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832659968/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763788.862524:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832659968:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763788.862528:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763788.862532:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832660096/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.862546:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763788.862548:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763788.862550:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763788.862552:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763788.862555:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763788.862558:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832660096:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763788.862562:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763788.862564:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763788.862566:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763788.862568:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763788.862572:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c8c00.
02000000:00000001:1.0:1787763788.862574:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763788.862577:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763788.862580:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763788.862583:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763788.862586:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832660096, portal 10
00000100:00000001:1.0:1787763788.862588:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763788.862589:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000100:00000040:1.0:1787763788.862596:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832660096/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763813 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.862604:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763788.862607:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763788.862609:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763788.862612:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832660096, offset 0
00000400:00000200:1.0:1787763788.862616:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763788.862620:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763788.862624:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763788.862627:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763788.862630:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763788.862633:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763788.862635:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763788.862640:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58392:58392:256:4294967295] 192.168.203.110@tcp LPNI seq info [58392:58392:8:4294967295]
00000400:00000200:1.0:1787763788.862645:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763788.862653:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763788.862658:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763788.862662:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763788.862666:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763788.862670:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787763788.862675:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763788.862678:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1787763788.862683:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763788.862698:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763788.862702:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763788.862707:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763788.862710:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763788.862712:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763788.862715:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763788.862719:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e1c0 x1874599832660096/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763813 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.862729:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763788.862732:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832660096/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763813 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.862739:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763788.862740:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763788.862744:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763788.862747:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763788.862750:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763788.862752:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316175893
00000100:00000001:1.0:1787763788.862755:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316177140
00000100:00000001:1.0:1787763788.862757:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316175893
00000100:00000001:1.0:1787763788.862760:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316177140
00000100:00000001:1.0:1787763788.862762:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763788.862765:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763788.862767:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763788.862771:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763788/real 1787763788]  req@ffff9751f876e1c0 x1874599832660096/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763788.862779:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763788.862780:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763788.862782:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763788.862785:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763788.862787:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763788.862789:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763788.862792:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832660096/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763788.862802:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832660096/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.862809:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763788.862810:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.862812:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763788.862814:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763788.862818:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832660096/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.862826:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763788.862830:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832660096/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.862838:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763788.862841:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.862843:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763788.862844:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.862846:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763788.862848:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763788.862851:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763788.862855:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763788.862858:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763788.862862:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832660096/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763788.862870:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832660096:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763788.862874:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763788.862879:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832660224/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.862888:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763788.862889:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763788.862892:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763788.862894:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763788.862897:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763788.862900:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832660224:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763788.862904:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763788.862906:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763788.862908:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763788.862910:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763788.862914:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763788.862917:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763788.862921:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763788.862923:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763788.862927:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763788.862931:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832660224, portal 4
00000100:00000001:1.0:1787763788.862933:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763788.862934:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
00000100:00000040:1.0:1787763788.862941:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832660224/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763813 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.862949:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763788.862953:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763788.862955:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763788.862958:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832660224, offset 0
00000400:00000200:1.0:1787763788.862962:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763788.862966:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763788.862970:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763788.862973:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763788.862976:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763788.862978:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763788.862981:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763788.862984:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58393:58393:256:4294967295] 192.168.203.110@tcp LPNI seq info [58393:58393:8:4294967295]
00000400:00000200:1.0:1787763788.862989:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763788.862996:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763788.863001:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763788.863004:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763788.863009:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763788.863012:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787763788.863016:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763788.863019:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1787763788.863024:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763788.863026:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763788.863029:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763788.863032:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763788.863035:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763788.863038:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763788.863040:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763788.863044:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876c000 x1874599832660224/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763813 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.863052:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763788.863056:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832660224/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763813 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.863064:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763788.863066:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763788.863069:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763788.863073:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763788.863076:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763788.863078:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316175910
00000100:00000001:1.0:1787763788.863081:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316177140
00000100:00000001:1.0:1787763788.863084:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316175910
00000100:00000001:1.0:1787763788.863086:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316177140
00000100:00000001:1.0:1787763788.863089:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763788.863092:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763788.863094:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763788.863098:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763788/real 1787763788]  req@ffff9751f876c000 x1874599832660224/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763788.863107:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763788.863109:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763788.863111:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763788.863113:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763788.863115:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763788.863118:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763788.863122:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832660224/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763788.863132:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832660224/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.863158:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763788.863160:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.863162:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763788.863164:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763788.863168:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832660224/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.863175:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763788.863179:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832660224/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.863187:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763788.863189:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.863191:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763788.863193:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.863195:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763788.863196:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763788.863199:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763788.863203:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763788.863205:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763788.863209:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832660224/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763788.863217:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832660224:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763788.863221:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.863223:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763788.863227:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832659968/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.863235:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763788.863237:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763788.863239:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763788.863242:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763788.863244:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763788.863245:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763788.863248:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763788.863249:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:1.0:1787763788.863252:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763788.863254:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763788.863255:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:1.0:1787763788.863258:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763788.863260:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.863262:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763788.863265:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832660096/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.863272:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763788.863273:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763788.863275:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c8c00.
02000000:00000001:1.0:1787763788.863277:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763788.863278:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763788.863280:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763788.863282:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763788.863284:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
02000000:00000001:1.0:1787763788.863287:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763788.863288:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763788.863290:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:1.0:1787763788.863292:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763788.863294:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.863296:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763788.863299:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832660224/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763813 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763788.863307:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763788.863308:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763788.863310:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763788.863312:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763788.863313:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763788.863315:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763788.863317:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763788.863319:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016c00.
02000000:00000001:1.0:1787763788.863321:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763788.863323:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763788.863324:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:1.0:1787763788.863327:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763788.863329:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.863331:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763788.863335:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763788.863337:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763788.863339:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763788.863342:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763789.500196:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763789.500206:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763789.500209:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763789.500214:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763789.500218:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763789.500222:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763789.500226:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763789.500228:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763789.500232:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763789.500273:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763789.500285:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763789.500299:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763789.500302:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763789.500311:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763789.500314:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763789.500380:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763789.500385:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763789.500389:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763789.884315:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763789.884329:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763789.884333:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763789.884343:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763789.884354:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763789.884360:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763789.884366:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763789.884371:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763789.884378:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763790.140282:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316177140, at 4316177152
00000100:00000010:3.0:1787763790.140307:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4de0.
00000100:00000040:2.0:1787763790.140408:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316177140, at 4316177152
00000100:00000010:2.0:1787763790.140423:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4960.
00010000:00000001:3.0:1787763790.524301:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763790.524325:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763790.524329:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763790.524334:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763790.524340:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763790.524341:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763790.524346:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763790.524347:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763790.524358:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763790.524361:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763790.524366:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763790.524367:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763790.524373:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763790.524376:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763790.524380:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763790.524474:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763790.524481:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763790.524487:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763790.908287:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763790.908299:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763790.908304:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763790.908313:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763790.908320:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763790.908326:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763790.908330:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763790.908334:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763790.908338:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763791.548208:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763791.548213:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763791.548217:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763791.548217:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763791.548220:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763791.548223:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763791.548224:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763791.548225:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763791.548228:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763791.548228:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763791.548231:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763791.548232:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763791.548234:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763791.548235:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787763791.548237:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763791.548238:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763791.548240:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763791.548240:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763791.548243:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763791.548245:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763791.548248:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763791.548272:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787763791.548275:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787763791.548278:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763791.932236:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763791.932244:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763791.932246:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763791.932252:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763791.932256:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763791.932259:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763791.932261:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763791.932264:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763791.932267:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763792.572266:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763792.572277:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763792.572282:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763792.572283:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763792.572293:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763792.572294:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763792.572305:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763792.572307:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763792.572311:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763792.572313:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763792.572318:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763792.572323:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763792.572324:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763792.572328:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763792.572331:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763792.572399:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763792.572406:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763792.572411:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763792.956255:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763792.956267:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763792.956271:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763792.956279:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763792.956287:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763792.956293:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763792.956298:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763792.956303:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763792.956308:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763793.596265:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763793.596278:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763793.596281:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763793.596289:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763793.596291:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763793.596296:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00100000:2.0:1787763793.596306:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763793.596306:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763793.596315:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787763793.596319:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763793.596320:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763793.596328:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763793.596329:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763793.596335:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763793.596337:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763793.596343:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763793.596344:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763793.596349:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763793.596350:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763793.596363:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763793.596365:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763793.596470:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763793.596476:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763793.596481:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763793.980276:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787763793.980288:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:0.0:1787763793.980289:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763793.980299:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1787763793.980302:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1787763793.980304:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763793.980310:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763793.980314:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00100000:3.0:1787763793.980323:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:0.0:1787763793.980328:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21508
00000100:00000001:3.0:1787763793.980333:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000400:00000200:0.0:1787763793.980343:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763793.980344:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763793.980353:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000400:00000200:0.0:1787763793.980356:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787763793.980361:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787763793.980363:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763793.980374:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763793.980374:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763793.980386:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763793.980391:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763793.980394:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763793.980399:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763793.980401:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763793.980405:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763793.980409:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763793.980412:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763793.980416:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763793.980421:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763793.980425:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763793.980430:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763793.980433:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763793.980438:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763793.980444:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763793.980448:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763793.980451:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763793.980455:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763793.980459:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763793.980461:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763793.980464:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763793.980474:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000020:00000040:0.0:1787763793.980478:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763793.980483:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763793.980484:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763793.980488:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763793.980491:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763793.980496:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763793.980507:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763793.980515:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763793.980527:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832660352/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763793.980542:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763793.980550:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763793.980560:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832660352/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763793.980599:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763793.980606:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763793.980612:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763793.980616:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763793.980621:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763793.980627:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763793.980636:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21508
00000100:00000001:1.0:1787763793.980638:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763793.980646:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000400:00000200:0.0:1787763793.980648:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763793.980667:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000400:00000200:0.0:1787763793.980684:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763793.980688:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763793.980693:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000040:1.0:1787763793.980696:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876de00 x1874599832660352/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763793.980697:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763793.980705:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763793.980713:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763793.980721:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763793.980725:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763793.980733:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:1.0:1787763793.980734:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:0.0:1787763793.980739:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000001:1.0:1787763793.980740:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763793.980749:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:1.0:1787763793.980752:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:0.0:1787763793.980756:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:1.0:1787763793.980759:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763793.980767:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763793.980768:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763793.980776:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:1.0:1787763793.980779:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832660352:192.168.203.110@tcp:250:kworker.0
00000100:00000040:0.0:1787763793.980784:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763793.980792:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1787763793.980794:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:1.0:1787763793.980800:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:0.0:1787763793.980801:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:1.0:1787763793.980806:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763793.980810:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:1.0:1787763793.980813:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1787763793.980816:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763793.980822:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:1.0:1787763793.980824:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763793.980830:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763793.980832:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1787763793.980842:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787763793.980843:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763793.980849:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000100:00000001:0.0:1787763793.980852:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:1.0:1787763793.980860:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000008:00100000:0.0:1787763793.980863:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000200:1.0:1787763793.980871:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832660352, portal 25
00000008:00000001:0.0:1787763793.980874:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763793.980878:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763793.980884:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763793.980884:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049525760 : -115096660025856 : ffff9751f876de00)
00000100:00000040:1.0:1787763793.980900:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876de00 x1874599832660352/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763818 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:0.0:1787763793.980901:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
00000020:00000040:0.0:1787763793.980913:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000100:00000001:1.0:1787763793.980918:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:0.0:1787763793.980926:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:1.0:1787763793.980927:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763793.980934:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
02000000:00000001:0.0:1787763793.980936:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787763793.980943:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832660352, offset 0
00000100:00000001:0.0:1787763793.980944:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763793.980951:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763793.980956:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763793.980959:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
02000000:00000010:0.0:1787763793.980967:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000400:00000200:1.0:1787763793.980975:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763793.980980:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763793.980988:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00100000:0.0:1787763793.980996:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832660480/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787763793.981001:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763793.981014:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:0.0:1787763793.981020:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1787763793.981024:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1787763793.981029:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763793.981035:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000040:0.0:1787763793.981044:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832660480/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787763793.981045:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58394:58394:256:4294967295] 192.168.203.110@tcp LPNI seq info [58394:58394:8:4294967295]
00000400:00000200:1.0:1787763793.981067:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763793.981083:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763793.981093:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787763793.981094:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00080000:0.0:1787763793.981107:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000800:00000200:1.0:1787763793.981115:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763793.981116:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763793.981123:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000010:1.0:1787763793.981129:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000100:00000001:0.0:1787763793.981130:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763793.981176:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21508
00000400:00000200:0.0:1787763793.981186:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:1.0:1787763793.981194:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000400:00000200:0.0:1787763793.981200:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000800:00000200:1.0:1787763793.981204:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000100:00080000:0.0:1787763793.981206:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763793.981217:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:1.1:1787763793.981225:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763793.981226:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.1:1787763793.981236:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000100:00000040:0.0:1787763793.981238:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763793.981248:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000100:1.0:1787763793.981254:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000020:00000001:0.0:1787763793.981257:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000010:1.0:1787763793.981260:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000020:00000040:0.0:1787763793.981262:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763793.981268:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:1.0:1787763793.981272:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:0.0:1787763793.981273:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763793.981284:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:1.0:1787763793.981289:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:0.0:1787763793.981292:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:1.0:1787763793.981298:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000100:00000040:0.0:1787763793.981301:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000010:1.0:1787763793.981304:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:0.0:1787763793.981306:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763793.981315:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763793.981316:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1787763793.981320:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763793.981328:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00000200:1.0:1787763793.981330:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876de00 x1874599832660352/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763818 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787763793.981337:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763793.981348:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763793.981356:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763793.981365:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763793.981365:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:0.0:1787763793.981372:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763793.981377:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876de00 x1874599832660352/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763818 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763793.981378:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763793.981385:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763793.981395:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763793.981403:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763793.981407:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763793.981412:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000100:00000001:1.0:1787763793.981413:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000040:0.0:1787763793.981421:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000001:1.0:1787763793.981425:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763793.981427:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763793.981432:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763793.981439:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:1.0:1787763793.981440:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787763793.981446:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1787763793.981448:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316182260
02000000:00000001:0.0:1787763793.981451:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763793.981461:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c46c0.
02000000:00000010:0.0:1787763793.981463:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787763793.981473:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763793.981477:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00100000:0.0:1787763793.981485:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832660608/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763793.981493:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316182260
00000100:00000010:1.0:1787763793.981504:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c47e0.
00000100:00000001:0.0:1787763793.981505:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763793.981514:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763793.981515:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000040:0.0:1787763793.981523:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832660608/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.981526:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763793.981536:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763793.981545:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763793.981558:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763793/real 1787763793]  req@ffff9751f876de00 x1874599832660352/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763793.981560:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763793.981575:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000001:1.0:1787763793.981579:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763793.981584:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763793.981589:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000100:00000040:0.0:1787763793.981591:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:1.0:1787763793.981594:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763793.981601:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00080000:0.0:1787763793.981605:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787763793.981606:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763793.981615:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876de00 x1874599832660352/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787763793.981622:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763793.981631:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763793.981640:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:1.0:1787763793.981642:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876de00 x1874599832660352/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763793.981675:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1787763793.981684:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00080000:0.0:1787763793.981685:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21508
00000100:00000001:1.0:1787763793.981691:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763793.981697:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000400:00000200:0.0:1787763793.981699:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763793.981701:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1787763793.981709:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00100000:1.0:1787763793.981716:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876de00 x1874599832660352/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787763793.981718:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763793.981732:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763793.981741:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787763793.981745:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763793.981749:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763793.981754:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:1.0:1787763793.981758:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876de00 x1874599832660352/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787763793.981760:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763793.981764:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763793.981770:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763793.981777:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000001:1.0:1787763793.981783:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000001:0.0:1787763793.981785:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:1.0:1787763793.981791:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1787763793.981794:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:1.0:1787763793.981797:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763793.981802:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763793.981806:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763793.981808:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763793.981814:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763793.981816:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00080000:1.0:1787763793.981823:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000040:0.0:1787763793.981824:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763793.981834:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00080000:1.0:1787763793.981835:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000020:00000040:0.0:1787763793.981845:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:1.0:1787763793.981845:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763793.981857:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876de00 x1874599832660352/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787763793.981864:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763793.981880:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763793.981884:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832660352:192.168.203.110@tcp:250:kworker.0
02000000:00000001:0.0:1787763793.981891:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787763793.981894:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763793.981899:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:0.0:1787763793.981903:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1787763793.981904:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876de00 x1874599832660352/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763793.981918:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763793.981923:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
00000100:00000001:0.0:1787763793.981927:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1787763793.981929:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000008:00100000:0.0:1787763793.981933:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
02000000:00000010:1.0:1787763793.981935:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca400.
00000008:00000001:0.0:1787763793.981945:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763793.981946:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000001:0.0:1787763793.981952:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763793.981953:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763793.981960:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763793.981964:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
00000100:00000010:0.0:1787763793.981967:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
02000000:00000010:1.0:1787763793.981968:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017800.
00000020:00000040:0.0:1787763793.981980:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787763793.981981:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763793.981986:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763793.981992:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876de00.
02000000:00000001:0.0:1787763793.981994:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1787763793.981999:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
02000000:00000001:0.0:1787763793.982001:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763793.982005:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763793.982011:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1787763793.982012:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787763793.982020:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1787763793.982026:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
02000000:00000001:0.0:1787763793.982028:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763793.982032:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763793.982039:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763793.982043:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
02000000:00000010:0.0:1787763793.982045:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:1.0:1787763793.982046:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763793.982054:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876da40 x1874599832660480/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763793.982059:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763793.982074:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763793.982079:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763793.982083:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832660736/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1787763793.982084:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763793.982089:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763793.982097:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763793.982110:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832660480:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763793.982123:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1787763793.982124:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1787763793.982126:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763793.982132:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763793.982169:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763793.982177:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763793.982184:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763793.982190:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763793.982192:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832660736/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787763793.982200:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763793.982208:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763793.982219:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763793.982226:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832660480, portal 10
00000100:00000001:1.0:1787763793.982231:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763793.982233:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763793.982235:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049524800 : -115096660026816 : ffff9751f876da40)
00000100:00000040:0.0:1787763793.982251:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21518)
00000100:00000040:1.0:1787763793.982254:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876da40 x1874599832660480/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763818 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.982266:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763793.982270:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763793.982274:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763793.982277:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832660480, offset 0
00000400:00000200:1.0:1787763793.982284:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763793.982289:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763793.982293:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763793.982298:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763793.982301:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763793.982304:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763793.982307:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763793.982311:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58395:58395:256:4294967295] 192.168.203.110@tcp LPNI seq info [58395:58395:8:4294967295]
00000400:00000200:1.0:1787763793.982317:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763793.982326:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763793.982331:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763793.982335:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763793.982341:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763793.982344:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787763793.982351:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763793.982354:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1787763793.982359:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763793.982361:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763793.982365:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763793.982370:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763793.982374:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763793.982376:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763793.982379:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763793.982384:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876da40 x1874599832660480/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763818 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.982394:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763793.982398:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876da40 x1874599832660480/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763818 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.982415:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763793.982417:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763793.982421:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763793.982426:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763793.982431:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763793.982433:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316181025
00000100:00000001:1.0:1787763793.982437:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316182260
00000100:00000001:1.0:1787763793.982440:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316181025
00000100:00000001:1.0:1787763793.982443:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316182260
00000100:00000001:1.0:1787763793.982445:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763793.982448:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763793.982450:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763793.982455:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763793/real 1787763793]  req@ffff9751f876da40 x1874599832660480/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763793.982464:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763793.982466:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763793.982468:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763793.982471:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763793.982473:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763793.982475:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763793.982479:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876da40 x1874599832660480/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763793.982488:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876da40 x1874599832660480/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.982496:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763793.982498:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763793.982500:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763793.982502:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763793.982507:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876da40 x1874599832660480/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.982516:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763793.982521:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876da40 x1874599832660480/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.982530:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763793.982533:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763793.982535:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763793.982536:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763793.982538:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763793.982552:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763793.982555:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763793.982559:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763793.982562:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763793.982567:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876da40 x1874599832660480/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763793.982576:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832660480:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763793.982581:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763793.982585:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cb40 x1874599832660608/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.982594:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763793.982596:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763793.982598:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763793.982600:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763793.982603:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763793.982606:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832660608:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763793.982610:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763793.982612:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763793.982614:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763793.982617:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763793.982621:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c8c00.
02000000:00000001:1.0:1787763793.982623:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763793.982627:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763793.982630:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763793.982634:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763793.982637:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832660608, portal 10
00000100:00000001:1.0:1787763793.982640:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763793.982642:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049520960 : -115096660030656 : ffff9751f876cb40)
00000100:00000040:1.0:1787763793.982672:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cb40 x1874599832660608/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763818 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.982681:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763793.982685:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763793.982688:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763793.982691:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832660608, offset 0
00000400:00000200:1.0:1787763793.982696:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763793.982700:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763793.982705:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763793.982709:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763793.982712:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763793.982715:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763793.982719:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763793.982723:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58396:58396:256:4294967295] 192.168.203.110@tcp LPNI seq info [58396:58396:8:4294967295]
00000400:00000200:1.0:1787763793.982728:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763793.982736:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763793.982741:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763793.982745:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763793.982750:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763793.982753:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787763793.982759:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763793.982762:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1787763793.982767:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763793.982770:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763793.982774:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763793.982778:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763793.982782:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763793.982784:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763793.982787:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763793.982791:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cb40 x1874599832660608/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763818 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.982801:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763793.982805:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cb40 x1874599832660608/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763818 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.982813:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763793.982815:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763793.982818:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763793.982821:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763793.982825:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763793.982827:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316181013
00000100:00000001:1.0:1787763793.982830:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316182260
00000100:00000001:1.0:1787763793.982833:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316181013
00000100:00000001:1.0:1787763793.982836:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316182260
00000100:00000001:1.0:1787763793.982838:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763793.982841:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763793.982843:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763793.982848:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763793/real 1787763793]  req@ffff9751f876cb40 x1874599832660608/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763793.982857:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763793.982858:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763793.982860:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763793.982863:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763793.982865:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763793.982868:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763793.982872:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cb40 x1874599832660608/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763793.982882:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cb40 x1874599832660608/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.982890:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763793.982892:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763793.982894:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763793.982895:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763793.982900:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cb40 x1874599832660608/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.982908:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763793.982912:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cb40 x1874599832660608/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.982920:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763793.982923:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763793.982925:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763793.982926:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763793.982929:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763793.982931:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763793.982934:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763793.982938:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763793.982941:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763793.982945:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cb40 x1874599832660608/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763793.982955:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832660608:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763793.982960:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763793.982962:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763793.982966:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876da40 x1874599832660480/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.982974:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763793.982976:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763793.982979:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763793.982981:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763793.982983:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763793.982985:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763793.982987:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763793.982989:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016c00.
02000000:00000001:1.0:1787763793.982994:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763793.982995:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763793.982997:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876da40.
00000100:00000001:1.0:1787763793.983000:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763793.983002:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763793.983004:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763793.983008:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cb40 x1874599832660608/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.983016:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763793.983017:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763793.983019:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c8c00.
02000000:00000001:1.0:1787763793.983022:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763793.983023:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763793.983024:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763793.983027:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763793.983028:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
02000000:00000001:1.0:1787763793.983030:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763793.983032:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763793.983033:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cb40.
00000100:00000001:1.0:1787763793.983037:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763793.983038:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763793.983040:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763793.983044:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763793.983046:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763793.983048:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763793.983051:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763793.983052:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763793.983057:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832660736/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.983066:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763793.983068:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763793.983070:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763793.983072:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763793.983075:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763793.983078:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832660736:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763793.983082:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763793.983085:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763793.983087:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763793.983089:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763793.983093:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c8c00.
02000000:00000001:1.0:1787763793.983095:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763793.983099:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763793.983101:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763793.983105:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763793.983109:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832660736, portal 4
00000100:00000001:1.0:1787763793.983112:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763793.983113:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
00000100:00000040:1.0:1787763793.983120:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832660736/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763818 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.983128:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763793.983131:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763793.983159:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763793.983164:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832660736, offset 0
00000400:00000200:1.0:1787763793.983169:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763793.983174:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763793.983178:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763793.983182:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763793.983186:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763793.983190:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763793.983193:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763793.983197:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58397:58397:256:4294967295] 192.168.203.110@tcp LPNI seq info [58397:58397:8:4294967295]
00000400:00000200:1.0:1787763793.983202:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763793.983210:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763793.983215:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763793.983219:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763793.983224:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763793.983227:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:1.1:1787763793.983233:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763793.983236:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:1.0:1787763793.983241:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763793.983243:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763793.983247:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763793.983251:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763793.983255:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763793.983257:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763793.983260:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763793.983265:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876c000 x1874599832660736/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763818 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.983274:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763793.983278:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832660736/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763818 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.983287:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763793.983289:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763793.983292:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763793.983295:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763793.983299:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763793.983302:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316181031
00000100:00000001:1.0:1787763793.983305:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316182260
00000100:00000001:1.0:1787763793.983308:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316181031
00000100:00000001:1.0:1787763793.983312:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316182260
00000100:00000001:1.0:1787763793.983315:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763793.983318:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763793.983320:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763793.983325:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763793/real 1787763793]  req@ffff9751f876c000 x1874599832660736/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763793.983333:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763793.983335:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763793.983337:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763793.983339:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763793.983341:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763793.983343:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763793.983347:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832660736/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763793.983357:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832660736/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.983365:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763793.983367:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763793.983369:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763793.983371:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763793.983375:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832660736/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.983383:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763793.983387:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832660736/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.983395:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763793.983398:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763793.983400:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763793.983401:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763793.983404:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763793.983407:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763793.983410:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763793.983414:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763793.983417:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763793.983421:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832660736/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763793.983430:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832660736:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763793.983435:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763793.983437:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763793.983440:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832660736/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763818 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763793.983448:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763793.983450:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763793.983452:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c8c00.
02000000:00000001:1.0:1787763793.983455:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763793.983457:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763793.983458:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763793.983461:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763793.983463:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:1.0:1787763793.983466:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763793.983468:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763793.983470:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:1.0:1787763793.983473:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763793.983474:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763793.983476:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763793.983480:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763793.983482:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763793.983484:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763793.983486:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763794.620248:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763794.620259:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763794.620271:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763794.620276:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763794.620285:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763794.620289:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763794.620302:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763794.620310:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763794.620313:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763794.620320:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763794.620325:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763794.620330:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763794.620335:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763794.620338:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763794.620343:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763794.620389:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763794.620398:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763794.620404:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763795.004211:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763795.004224:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763795.004227:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763795.004235:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763795.004240:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763795.004244:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763795.004247:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763795.004251:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763795.004257:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763795.260234:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316182260, at 4316182272
00000100:00000010:3.0:1787763795.260255:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c47e0.
00000100:00000040:2.0:1787763795.260322:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316182260, at 4316182272
00000100:00000010:2.0:1787763795.260337:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c46c0.
00000100:00000001:2.0:1787763795.644263:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763795.644271:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763795.644273:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763795.644278:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763795.644281:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763795.644284:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763795.644286:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763795.644287:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763795.644289:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763795.644292:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763795.644293:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763795.644299:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763795.644303:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763795.644305:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763795.644308:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763795.644311:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763795.644312:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763795.644313:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763795.644315:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763795.644333:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763795.644335:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763795.644387:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763795.644392:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763795.644396:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763796.028343:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763796.028351:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763796.028354:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763796.028359:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763796.028364:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763796.028367:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763796.028370:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763796.028372:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763796.028374:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763796.668264:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763796.668273:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763796.668280:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763796.668283:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763796.668291:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763796.668292:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763796.668298:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763796.668299:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763796.668306:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763796.668308:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763796.668317:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763796.668321:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763796.668325:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763796.668328:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763796.668331:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763796.668406:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763796.668413:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763796.668418:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763797.052248:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763797.052264:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763797.052269:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763797.052277:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763797.052284:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763797.052291:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763797.052296:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763797.052300:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763797.052305:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763797.692296:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763797.692305:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763797.692312:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763797.692314:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763797.692319:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763797.692322:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763797.692336:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763797.692350:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763797.692354:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763797.692363:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763797.692369:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763797.692375:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763797.692379:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763797.692383:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763797.692389:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763797.692407:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763797.692413:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763797.692421:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763798.076331:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763798.076343:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763798.076360:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763798.076371:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763798.076379:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763798.076385:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763798.076390:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763798.076395:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763798.076401:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763798.716260:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763798.716269:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763798.716272:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763798.716280:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763798.716283:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763798.716285:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763798.716289:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763798.716294:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763798.716297:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763798.716299:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763798.716307:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763798.716313:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763798.716318:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763798.716321:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763798.716326:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763798.716360:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763798.716367:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763798.716372:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763799.100324:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763799.100337:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763799.100343:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763799.100354:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:2.0:1787763799.100361:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787763799.100363:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763799.100375:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1787763799.100378:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787763799.100386:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763799.100388:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1787763799.100395:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1787763799.100399:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763799.100404:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763799.100411:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763799.100427:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21513
00000400:00000200:2.0:1787763799.100442:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763799.100453:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763799.100461:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763799.100469:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763799.100475:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763799.100480:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763799.100486:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763799.100494:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763799.100498:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787763799.100503:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763799.100510:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787763799.100516:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787763799.100520:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787763799.100527:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763799.100531:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763799.100534:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763799.100538:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763799.100544:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787763799.100550:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763799.100554:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763799.100559:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763799.100565:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763799.100570:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763799.100574:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787763799.100578:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763799.100590:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658780.
00000020:00000040:2.0:1787763799.100596:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787763799.100602:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763799.100605:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763799.100609:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763799.100645:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763799.100654:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763799.100669:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bbc00.
00000100:00000001:2.0:1787763799.100680:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763799.100695:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658780 x1874599832660864/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763799.100726:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763799.100733:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763799.100743:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000744875b5] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658780 x1874599832660864/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763799.100800:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763799.100807:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763799.100815:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787763799.100818:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763799.100822:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763799.100827:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763799.100833:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21513
00000400:00000200:2.0:1787763799.100841:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763799.100847:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763799.100852:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763799.100858:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763799.100862:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763799.100866:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763799.100870:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763799.100875:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763799.100877:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:2.0:1787763799.100881:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763799.100886:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:2.0:1787763799.100890:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:2.0:1787763799.100894:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:2.0:1787763799.100900:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763799.100904:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763799.100908:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763799.100912:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763799.100917:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:2.0:1787763799.100923:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763799.100927:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763799.100931:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763799.100939:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763799.100943:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763799.100947:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787763799.100954:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787763799.100961:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763799.100965:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763799.100974:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000020:00000040:2.0:1787763799.100979:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787763799.100984:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763799.100986:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763799.100990:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763799.100993:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763799.100996:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763799.101005:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5ba000.
00000100:00000001:2.0:1787763799.101011:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763799.101021:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832660992/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763799.101037:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763799.101041:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763799.101061:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832660992/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763799.101077:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763799.101083:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763799.101090:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1787763799.101093:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763799.101096:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763799.101101:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763799.101106:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21513
00000400:00000200:2.0:1787763799.101113:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763799.101119:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763799.101123:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763799.101129:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763799.101205:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763799.101213:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763799.101220:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763799.101224:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763799.101227:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787763799.101230:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763799.101234:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:2.0:1787763799.101239:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:2.0:1787763799.101244:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:2.0:1787763799.101250:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763799.101255:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763799.101261:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763799.101266:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763799.101272:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:2.0:1787763799.101279:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763799.101284:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763799.101289:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763799.101327:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763799.101332:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763799.101336:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787763799.101340:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787763799.101346:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763799.101350:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763799.101358:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000020:00000040:2.0:1787763799.101363:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787763799.101367:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763799.101370:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763799.101373:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763799.101376:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763799.101380:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763799.101400:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bb800.
00000100:00000001:2.0:1787763799.101408:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763799.101420:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832661120/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763799.101446:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763799.101450:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763799.101459:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832661120/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763799.101471:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763799.101475:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1787763799.101481:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:2.0:1787763799.101486:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763799.101490:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787763799.101492:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763799.101495:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763799.101497:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763799.101501:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21513
00000400:00000200:2.0:1787763799.101508:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763799.101513:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763799.101516:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763799.101521:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763799.101523:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763799.101526:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763799.101529:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763799.101532:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763799.101534:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787763799.101537:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763799.101540:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787763799.101542:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:2.0:1787763799.101546:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:2.0:1787763799.101552:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763799.101557:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763799.101561:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763799.101565:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763799.101570:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:2.0:1787763799.101576:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763799.101580:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763799.101584:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763799.101590:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763799.101594:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763799.101598:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787763799.101603:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1787763799.101608:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763799.101611:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763799.101642:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
00000020:00000040:2.0:1787763799.101648:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:2.0:1787763799.101653:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763799.101657:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763799.101660:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763799.101664:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763799.101668:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763799.101684:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5b9800.
00000100:00000001:2.0:1787763799.101692:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763799.101699:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00100000:2.0:1787763799.101706:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832661248/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763799.101709:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763799.101715:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763799.101731:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832660864/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763799.101733:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763799.101744:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763799.101757:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763799.101763:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763799.101763:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832661248/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787763799.101773:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763799.101783:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763799.101787:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763799.101793:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763799.101797:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21523)
00000100:00100000:1.0:1787763799.101825:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832660864:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763799.101838:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763799.101845:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763799.101849:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763799.101854:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763799.101863:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763799.101868:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763799.101878:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763799.101886:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763799.101898:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763799.101908:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832660864, portal 25
00000100:00000001:1.0:1787763799.101914:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763799.101917:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000040:1.0:1787763799.101930:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832660864/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763824 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.101947:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763799.101953:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763799.101958:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763799.101966:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832660864, offset 0
00000400:00000200:1.0:1787763799.101978:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763799.101993:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763799.102005:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763799.102013:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763799.102018:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763799.102023:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763799.102027:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763799.102032:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58398:58398:256:4294967295] 192.168.203.110@tcp LPNI seq info [58398:58398:8:4294967295]
00000400:00000200:1.0:1787763799.102039:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763799.102052:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763799.102059:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763799.102067:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763799.102076:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763799.102082:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787763799.102089:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763799.102094:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787763799.102100:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763799.102103:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763799.102109:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763799.102114:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763799.102121:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763799.102124:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763799.102129:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763799.102187:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832660864/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763824 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.102202:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763799.102208:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832660864/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763824 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.102217:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763799.102220:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763799.102229:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763799.102235:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763799.102239:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316187381
00000100:00000010:1.0:1787763799.102247:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4180.
00000100:00000001:1.0:1787763799.102258:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787763799.102268:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316187381
00000100:00000010:1.0:1787763799.102273:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4d20.
00000100:00000001:1.0:1787763799.102279:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763799.102284:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763799.102287:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763799.102291:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763799.102297:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763799/real 1787763799]  req@ffff9751f8658780 x1874599832660864/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763799.102308:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763799.102311:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763799.102314:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763799.102317:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763799.102320:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763799.102323:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763799.102327:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832660864/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763799.102339:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832660864/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.102347:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763799.102349:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763799.102352:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763799.102354:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763799.102359:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832660864/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.102368:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763799.102374:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832660864/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.102383:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763799.102386:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763799.102389:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763799.102391:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763799.102394:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763799.102396:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763799.102400:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763799.102405:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763799.102408:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763799.102413:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832660864/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763799.102423:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832660864:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763799.102429:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763799.102434:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832660992/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.102443:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763799.102445:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763799.102448:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763799.102450:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763799.102453:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763799.102457:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832660992:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763799.102461:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763799.102464:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763799.102466:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763799.102469:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763799.102473:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763799.102476:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763799.102480:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763799.102484:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763799.102488:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763799.102492:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832660992, portal 10
00000100:00000001:1.0:1787763799.102494:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763799.102497:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:1.0:1787763799.102505:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832660992/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763824 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.102514:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763799.102517:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763799.102519:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763799.102522:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832660992, offset 0
00000400:00000200:1.0:1787763799.102530:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763799.102537:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763799.102545:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763799.102551:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763799.102557:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763799.102561:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763799.102566:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763799.102572:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58399:58399:256:4294967295] 192.168.203.110@tcp LPNI seq info [58399:58399:8:4294967295]
00000400:00000200:1.0:1787763799.102582:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763799.102595:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763799.102605:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763799.102613:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763799.102646:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763799.102650:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787763799.102656:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763799.102660:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787763799.102665:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763799.102668:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763799.102672:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763799.102677:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763799.102682:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763799.102684:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763799.102688:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763799.102693:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832660992/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763824 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.102703:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763799.102709:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832660992/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763824 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.102717:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763799.102719:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763799.102724:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763799.102729:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763799.102734:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763799.102737:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316186145
00000100:00000001:1.0:1787763799.102741:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316187381
00000100:00000001:1.0:1787763799.102744:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316186145
00000100:00000001:1.0:1787763799.102746:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316187381
00000100:00000001:1.0:1787763799.102749:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763799.102752:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763799.102754:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763799.102759:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763799/real 1787763799]  req@ffff9751f865ad00 x1874599832660992/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763799.102768:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763799.102770:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763799.102772:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763799.102775:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763799.102777:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763799.102780:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763799.102784:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832660992/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763799.102794:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832660992/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.102801:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763799.102803:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763799.102805:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763799.102807:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763799.102813:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832660992/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.102822:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763799.102826:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832660992/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.102835:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763799.102838:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763799.102840:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763799.102841:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763799.102844:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763799.102857:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763799.102860:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763799.102864:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763799.102867:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763799.102871:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832660992/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763799.102881:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832660992:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763799.102886:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763799.102890:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832661120/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.102900:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763799.102903:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763799.102907:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763799.102911:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763799.102915:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763799.102921:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832661120:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763799.102928:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763799.102932:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763799.102935:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763799.102939:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763799.102946:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb800.
02000000:00000001:1.0:1787763799.102950:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763799.102957:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763799.102962:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763799.102968:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763799.102974:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832661120, portal 10
00000100:00000001:1.0:1787763799.102979:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763799.102982:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787763799.102994:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832661120/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763824 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.103009:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763799.103012:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763799.103016:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763799.103019:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832661120, offset 0
00000400:00000200:1.0:1787763799.103025:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763799.103030:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763799.103035:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763799.103039:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763799.103043:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763799.103046:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763799.103049:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763799.103053:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58400:58400:256:4294967295] 192.168.203.110@tcp LPNI seq info [58400:58400:8:4294967295]
00000400:00000200:1.0:1787763799.103058:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763799.103067:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763799.103072:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763799.103076:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763799.103081:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763799.103084:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787763799.103090:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763799.103093:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787763799.103099:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763799.103102:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763799.103106:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763799.103111:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763799.103115:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763799.103117:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763799.103120:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763799.103125:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832661120/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763824 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.103170:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763799.103176:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832661120/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763824 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.103187:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763799.103189:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763799.103194:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763799.103198:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763799.103202:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763799.103205:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316186134
00000100:00000001:1.0:1787763799.103208:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316187381
00000100:00000001:1.0:1787763799.103211:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316186134
00000100:00000001:1.0:1787763799.103214:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316187381
00000100:00000001:1.0:1787763799.103216:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763799.103219:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763799.103221:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763799.103226:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763799/real 1787763799]  req@ffff9751f8659a40 x1874599832661120/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763799.103235:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763799.103237:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763799.103239:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763799.103242:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763799.103244:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763799.103248:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763799.103251:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832661120/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763799.103262:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832661120/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.103270:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763799.103271:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763799.103273:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763799.103275:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763799.103280:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832661120/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.103288:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763799.103292:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832661120/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.103301:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763799.103304:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763799.103306:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763799.103307:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763799.103310:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763799.103312:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763799.103315:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763799.103319:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763799.103322:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763799.103326:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832661120/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763799.103335:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832661120:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763799.103340:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763799.103344:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763799.103348:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832660864/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.103357:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763799.103360:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763799.103363:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763799.103367:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763799.103369:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763799.103371:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763799.103375:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763799.103378:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:1.0:1787763799.103382:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763799.103385:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763799.103387:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:1.0:1787763799.103391:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763799.103392:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763799.103394:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763799.103398:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832660992/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.103406:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763799.103408:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763799.103410:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763799.103412:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763799.103414:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763799.103416:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763799.103418:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763799.103420:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:1.0:1787763799.103422:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763799.103423:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763799.103425:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:1.0:1787763799.103428:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763799.103429:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763799.103431:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763799.103435:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832661120/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.103442:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763799.103444:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763799.103446:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb800.
02000000:00000001:1.0:1787763799.103449:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763799.103450:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763799.103452:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763799.103454:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763799.103456:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:1.0:1787763799.103458:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763799.103459:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763799.103461:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787763799.103463:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763799.103465:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763799.103467:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763799.103475:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763799.103477:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763799.103480:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763799.103482:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763799.103483:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763799.103487:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832661248/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.103496:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763799.103497:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763799.103500:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763799.103502:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763799.103505:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763799.103509:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832661248:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763799.103513:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763799.103515:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763799.103517:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763799.103519:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763799.103524:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb800.
02000000:00000001:1.0:1787763799.103527:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763799.103534:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763799.103539:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763799.103546:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763799.103552:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832661248, portal 4
00000100:00000001:1.0:1787763799.103557:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763799.103559:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787763799.103571:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832661248/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763824 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.103586:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763799.103591:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763799.103597:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763799.103602:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832661248, offset 0
00000400:00000200:1.0:1787763799.103611:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763799.103642:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763799.103648:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763799.103652:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763799.103656:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763799.103659:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763799.103662:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763799.103666:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58401:58401:256:4294967295] 192.168.203.110@tcp LPNI seq info [58401:58401:8:4294967295]
00000400:00000200:1.0:1787763799.103672:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763799.103680:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763799.103686:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763799.103691:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763799.103696:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763799.103700:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787763799.103706:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763799.103709:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787763799.103714:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763799.103717:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763799.103721:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763799.103726:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763799.103730:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763799.103732:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763799.103736:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763799.103741:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832661248/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763824 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.103760:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763799.103764:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832661248/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763824 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.103773:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763799.103775:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763799.103779:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763799.103783:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763799.103787:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763799.103789:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316186151
00000100:00000001:1.0:1787763799.103793:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316187381
00000100:00000001:1.0:1787763799.103796:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316186151
00000100:00000001:1.0:1787763799.103799:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316187381
00000100:00000001:1.0:1787763799.103801:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763799.103804:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763799.103806:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763799.103811:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763799/real 1787763799]  req@ffff9751f8658000 x1874599832661248/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763799.103820:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763799.103822:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763799.103824:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763799.103827:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763799.103829:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763799.103832:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763799.103836:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832661248/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763799.103846:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832661248/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.103854:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763799.103856:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763799.103859:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763799.103861:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763799.103865:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832661248/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.103873:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763799.103878:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832661248/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.103886:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763799.103889:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763799.103891:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763799.103892:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763799.103895:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763799.103896:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763799.103901:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763799.103907:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763799.103910:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763799.103918:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832661248/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763799.103934:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832661248:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763799.103941:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763799.103945:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763799.103952:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832661248/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763824 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763799.103966:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763799.103969:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763799.103973:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb800.
02000000:00000001:1.0:1787763799.103978:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763799.103982:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763799.103985:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763799.103989:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763799.103993:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:1.0:1787763799.103998:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763799.104000:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763799.104003:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:1.0:1787763799.104007:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763799.104009:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763799.104011:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763799.104016:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763799.104018:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763799.104020:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763799.104023:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763799.740276:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763799.740278:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763799.740289:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763799.740290:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763799.740297:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763799.740304:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763799.740307:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763799.740310:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763799.740315:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763799.740317:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763799.740326:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763799.740331:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763799.740333:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763799.740336:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763799.740341:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763799.740478:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763799.740486:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763799.740491:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763800.124268:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763800.124276:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763800.124280:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763800.124289:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763800.124297:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763800.124304:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763800.124309:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763800.124315:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763800.124322:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763800.380197:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316187381, at 4316187392
00000100:00000010:3.0:1787763800.380216:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4d20.
00000100:00000040:2.0:1787763800.380242:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316187381, at 4316187392
00000100:00000010:2.0:1787763800.380250:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4180.
00010000:00000001:3.0:1787763800.764256:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763800.764265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763800.764275:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763800.764276:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763800.764278:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763800.764287:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763800.764289:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763800.764294:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763800.764296:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763800.764307:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763800.764314:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763800.764319:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763800.764323:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763800.764327:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763800.764331:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763800.764388:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763800.764395:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763800.764400:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763801.148293:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763801.148303:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763801.148306:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763801.148314:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763801.148319:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763801.148323:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763801.148327:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763801.148330:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763801.148334:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763801.788219:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763801.788222:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763801.788226:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763801.788228:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763801.788232:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763801.788233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763801.788236:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763801.788236:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763801.788242:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763801.788243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763801.788246:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763801.788247:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763801.788249:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763801.788250:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787763801.788253:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763801.788254:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763801.788257:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763801.788258:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763801.788261:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763801.788263:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763801.788265:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763801.788306:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763801.788311:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763801.788316:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763802.172290:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763802.172301:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763802.172304:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763802.172310:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763802.172315:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763802.172318:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763802.172321:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763802.172324:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763802.172327:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763802.812208:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763802.812216:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763802.812218:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763802.812224:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763802.812228:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763802.812231:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763802.812234:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763802.812236:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763802.812241:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763802.812283:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763802.812291:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763802.812299:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763802.812301:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763802.812306:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763802.812308:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763802.812368:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763802.812372:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763802.812375:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763803.196291:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763803.196303:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763803.196305:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763803.196311:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763803.196315:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763803.196319:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763803.196322:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763803.196325:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763803.196328:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763803.836289:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763803.836298:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763803.836300:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763803.836304:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763803.836312:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00100000:2.0:1787763803.836313:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763803.836320:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787763803.836323:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763803.836329:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763803.836331:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763803.836351:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763803.836352:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787763803.836367:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763803.836367:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763803.836399:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763803.836411:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763803.836412:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763803.836428:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763803.836432:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763803.836439:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763803.836442:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763803.836539:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763803.836546:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763803.836550:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787763804.220252:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763804.220262:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787763804.220265:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763804.220269:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763804.220275:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763804.220286:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21518
00000400:00000200:2.0:1787763804.220295:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763804.220302:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763804.220306:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763804.220309:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763804.220312:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763804.220314:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763804.220316:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763804.220320:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763804.220321:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787763804.220323:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763804.220327:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787763804.220330:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787763804.220332:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787763804.220337:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763804.220340:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787763804.220341:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000040:2.0:1787763804.220344:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763804.220351:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:0.0:1787763804.220354:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000020:00000040:2.0:1787763804.220355:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00000001:0.0:1787763804.220362:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763804.220369:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00100000:0.0:1787763804.220375:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763804.220376:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763804.220392:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
02000000:00000001:2.0:1787763804.220392:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787763804.220406:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1787763804.220406:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763804.220422:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763804.220424:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787763804.220426:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763804.220434:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000100:00000001:0.0:1787763804.220438:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:2.0:1787763804.220438:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
00000100:00000001:0.0:1787763804.220447:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1787763804.220448:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763804.220456:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763804.220459:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763804.220463:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763804.220467:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763804.220473:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763804.220481:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bb800.
00000100:00000001:2.0:1787763804.220487:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763804.220495:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832661376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763804.220504:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763804.220508:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763804.220514:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832661376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763804.220534:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763804.220537:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763804.220541:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787763804.220543:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763804.220544:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763804.220549:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763804.220555:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21518
00000400:00000200:2.0:1787763804.220559:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763804.220564:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763804.220567:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763804.220574:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763804.220576:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763804.220578:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787763804.220595:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763804.220599:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763804.220599:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763804.220605:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763804.220608:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000040:1.0:1787763804.220611:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832661376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787763804.220611:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:2.0:1787763804.220616:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763804.220620:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000001:1.0:1787763804.220624:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:2.0:1787763804.220624:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:1.0:1787763804.220627:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763804.220628:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:1.0:1787763804.220633:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:2.0:1787763804.220634:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:1.0:1787763804.220637:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763804.220637:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763804.220640:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:1.0:1787763804.220641:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763804.220644:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763804.220648:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00100000:1.0:1787763804.220649:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832661376:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763804.220655:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:2.0:1787763804.220655:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763804.220660:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763804.220661:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763804.220663:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763804.220665:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763804.220668:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1787763804.220671:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763804.220674:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763804.220675:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb800.
00000100:00000001:2.0:1787763804.220676:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1787763804.220681:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:2.0:1787763804.220683:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787763804.220686:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763804.220691:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763804.220692:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763804.220697:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000100:00000010:2.0:1787763804.220700:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
00000020:00000040:2.0:1787763804.220703:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000010:1.0:1787763804.220704:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
02000000:00000001:2.0:1787763804.220707:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763804.220710:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763804.220711:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:1.0:1787763804.220713:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832661376, portal 25
02000000:00000001:2.0:1787763804.220715:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1787763804.220716:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:2.0:1787763804.220720:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763804.220721:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
02000000:00000010:2.0:1787763804.220728:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5ba000.
00000100:00000040:1.0:1787763804.220731:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832661376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763829 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763804.220734:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763804.220744:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00100000:2.0:1787763804.220747:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832661504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1787763804.220749:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763804.220754:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000001:2.0:1787763804.220759:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763804.220762:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787763804.220764:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832661376, offset 0
00000100:00000040:2.0:1787763804.220768:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832661504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763804.220777:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763804.220778:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00080000:2.0:1787763804.220784:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763804.220789:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:1.0:1787763804.220793:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763804.220793:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763804.220797:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763804.220803:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1787763804.220809:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00080000:2.0:1787763804.220811:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21518
00000400:00000200:2.0:1787763804.220820:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763804.220821:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763804.220828:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:1.0:1787763804.220829:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00080000:2.0:1787763804.220831:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1787763804.220836:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000040:2.0:1787763804.220841:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:1.0:1787763804.220843:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:2.0:1787763804.220846:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787763804.220851:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58402:58402:256:4294967295] 192.168.203.110@tcp LPNI seq info [58402:58402:8:4294967295]
00000100:00000040:2.0:1787763804.220852:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763804.220858:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763804.220862:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:1.0:1787763804.220867:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:2.0:1787763804.220867:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787763804.220871:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763804.220874:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:2.0:1787763804.220876:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:2.0:1787763804.220878:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:2.0:1787763804.220882:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763804.220886:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787763804.220890:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:2.0:1787763804.220892:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763804.220896:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763804.220899:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:2.0:1787763804.220903:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000800:00000200:1.0:1787763804.220905:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787763804.220906:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763804.220909:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763804.220911:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000010:1.0:1787763804.220912:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
02000000:00000001:2.0:1787763804.220913:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763804.220917:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787763804.220919:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787763804.220921:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787763804.220924:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787763804.220924:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787763804.220930:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000100:00000010:2.0:1787763804.220931:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000020:00000040:2.0:1787763804.220937:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787763804.220941:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:1.1:1787763804.220942:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:2.0:1787763804.220943:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763804.220947:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:1.1:1787763804.220948:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
02000000:00000001:2.0:1787763804.220949:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763804.220953:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:1.0:1787763804.220957:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763804.220960:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
02000000:00000010:2.0:1787763804.220960:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bbc00.
00000100:00000001:2.0:1787763804.220965:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763804.220968:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00100000:2.0:1787763804.220970:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832661632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787763804.220976:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763804.220984:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000100:00000001:2.0:1787763804.220984:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763804.220988:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763804.220989:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000040:2.0:1787763804.220993:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832661632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.220995:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763804.221003:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832661376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763829 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763804.221004:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763804.221009:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1787763804.221017:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:2.0:1787763804.221021:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787763804.221022:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00080000:2.0:1787763804.221026:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000040:1.0:1787763804.221028:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832661376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763829 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763804.221029:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763804.221032:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763804.221037:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763804.221042:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21518
00000100:00000001:1.0:1787763804.221043:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763804.221046:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000400:00000200:2.0:1787763804.221047:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763804.221051:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763804.221053:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787763804.221056:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763804.221056:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763804.221058:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763804.221062:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:1.0:1787763804.221064:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763804.221065:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787763804.221069:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316192499
00000020:00000001:2.0:1787763804.221069:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763804.221070:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787763804.221074:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000010:1.0:1787763804.221076:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c48a0.
00000020:00000040:2.0:1787763804.221076:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787763804.221078:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:2.0:1787763804.221081:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:1.0:1787763804.221084:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000040:2.0:1787763804.221085:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763804.221089:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763804.221091:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763804.221093:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787763804.221097:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316192499
00000020:00000040:2.0:1787763804.221097:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000010:1.0:1787763804.221103:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4900.
00000100:00080000:2.0:1787763804.221104:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763804.221108:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763804.221109:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
02000000:00000001:2.0:1787763804.221113:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787763804.221116:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763804.221116:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763804.221121:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763804.221122:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763804.221124:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1787763804.221125:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787763804.221129:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00100000:1.0:1787763804.221131:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763804/real 1787763804]  req@ffff9751f86583c0 x1874599832661376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763804.221170:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763804.221171:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000008:00000001:2.0:1787763804.221173:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763804.221174:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000100:00000001:2.0:1787763804.221181:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763804.221182:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763804.221188:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000010:2.0:1787763804.221190:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000100:00000001:1.0:1787763804.221192:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763804.221197:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832661376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787763804.221197:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:2.0:1787763804.221205:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763804.221207:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787763804.221209:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832661376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763804.221210:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763804.221214:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763804.221217:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763804.221219:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000010:2.0:1787763804.221223:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5b8c00.
00000100:00000001:1.0:1787763804.221224:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763804.221228:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763804.221230:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763804.221230:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763804.221245:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832661376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763804.221245:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832661760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763804.221276:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763804.221279:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763804.221280:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763804.221292:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832661376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787763804.221297:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832661760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.221303:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763804.221312:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763804.221314:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763804.221316:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763804.221322:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763804.221325:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21528)
00000100:00000001:1.0:1787763804.221327:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763804.221334:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763804.221339:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763804.221350:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763804.221357:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763804.221367:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832661376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763804.221376:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832661376:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763804.221388:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763804.221391:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763804.221397:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832661376/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.221404:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763804.221406:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763804.221409:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb800.
02000000:00000001:1.0:1787763804.221412:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763804.221413:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763804.221415:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763804.221418:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763804.221420:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:1.0:1787763804.221426:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763804.221427:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763804.221429:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:1.0:1787763804.221432:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763804.221433:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763804.221434:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763804.221440:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763804.221442:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763804.221443:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763804.221445:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763804.221446:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763804.221448:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832661504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.221457:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763804.221458:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763804.221462:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763804.221465:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763804.221467:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763804.221471:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832661504:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763804.221473:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763804.221475:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763804.221477:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763804.221478:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763804.221481:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb800.
02000000:00000001:1.0:1787763804.221482:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763804.221487:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763804.221490:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763804.221494:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763804.221498:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832661504, portal 10
00000100:00000001:1.0:1787763804.221501:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763804.221503:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787763804.221508:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832661504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763829 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.221513:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763804.221516:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763804.221518:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763804.221521:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832661504, offset 0
00000400:00000200:1.0:1787763804.221525:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763804.221530:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763804.221534:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763804.221537:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763804.221540:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763804.221542:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763804.221545:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763804.221548:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58403:58403:256:4294967295] 192.168.203.110@tcp LPNI seq info [58403:58403:8:4294967295]
00000400:00000200:1.0:1787763804.221553:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763804.221561:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763804.221565:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763804.221567:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763804.221571:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763804.221573:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787763804.221576:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763804.221577:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787763804.221596:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763804.221598:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763804.221601:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763804.221603:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763804.221606:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763804.221607:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763804.221609:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763804.221612:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832661504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763829 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.221618:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763804.221620:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832661504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763829 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.221629:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763804.221630:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763804.221633:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763804.221636:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763804.221638:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763804.221640:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316191264
00000100:00000001:1.0:1787763804.221642:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316192499
00000100:00000001:1.0:1787763804.221644:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316191264
00000100:00000001:1.0:1787763804.221645:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316192499
00000100:00000001:1.0:1787763804.221646:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763804.221648:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763804.221649:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763804.221651:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763804/real 1787763804]  req@ffff9751f8658000 x1874599832661504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763804.221656:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763804.221657:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763804.221659:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763804.221660:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763804.221661:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763804.221663:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763804.221665:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832661504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763804.221670:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832661504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.221674:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763804.221674:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763804.221676:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763804.221676:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763804.221679:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832661504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.221684:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763804.221686:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832661504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.221691:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763804.221692:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763804.221693:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763804.221694:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763804.221695:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763804.221707:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763804.221710:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763804.221713:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763804.221715:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763804.221719:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832661504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763804.221727:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832661504:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763804.221732:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763804.221735:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832661632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.221742:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763804.221744:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763804.221746:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763804.221748:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763804.221750:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763804.221754:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832661632:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763804.221757:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763804.221759:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763804.221761:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763804.221763:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763804.221768:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763804.221770:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763804.221773:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763804.221776:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763804.221780:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763804.221783:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832661632, portal 10
00000100:00000001:1.0:1787763804.221785:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763804.221787:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787763804.221794:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832661632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763829 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.221801:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763804.221803:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763804.221806:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763804.221808:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832661632, offset 0
00000400:00000200:1.0:1787763804.221812:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763804.221816:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763804.221819:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763804.221821:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763804.221823:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763804.221824:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763804.221826:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763804.221828:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58404:58404:256:4294967295] 192.168.203.110@tcp LPNI seq info [58404:58404:8:4294967295]
00000400:00000200:1.0:1787763804.221831:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763804.221835:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763804.221839:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763804.221841:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763804.221843:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763804.221845:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787763804.221848:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763804.221850:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787763804.221853:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763804.221855:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763804.221857:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763804.221860:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763804.221863:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763804.221865:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763804.221868:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763804.221872:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832661632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763829 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.221879:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763804.221883:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832661632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763829 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.221890:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763804.221892:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763804.221895:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763804.221898:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763804.221902:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763804.221904:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316191252
00000100:00000001:1.0:1787763804.221907:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316192499
00000100:00000001:1.0:1787763804.221909:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316191252
00000100:00000001:1.0:1787763804.221911:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316192499
00000100:00000001:1.0:1787763804.221913:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763804.221916:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763804.221917:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763804.221920:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763804/real 1787763804]  req@ffff9751f8659a40 x1874599832661632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763804.221926:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763804.221927:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763804.221928:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763804.221930:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763804.221932:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763804.221934:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763804.221936:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832661632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763804.221941:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832661632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.221945:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763804.221946:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763804.221947:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763804.221948:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763804.221950:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832661632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.221955:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763804.221957:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832661632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.221962:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763804.221963:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763804.221964:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763804.221965:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763804.221966:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763804.221967:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763804.221969:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763804.221970:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763804.221972:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763804.221974:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832661632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763804.221979:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832661632:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763804.221981:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763804.221984:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832661760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.221988:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763804.221989:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763804.221991:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763804.221992:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763804.221993:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763804.221995:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832661760:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763804.221997:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763804.221999:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763804.222000:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763804.222001:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763804.222004:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763804.222005:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763804.222008:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763804.222009:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763804.222012:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763804.222014:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832661760, portal 4
00000100:00000001:1.0:1787763804.222015:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763804.222016:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:1.0:1787763804.222020:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832661760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763829 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.222024:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763804.222026:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763804.222027:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763804.222029:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832661760, offset 0
00000400:00000200:1.0:1787763804.222031:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763804.222034:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763804.222036:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763804.222038:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763804.222040:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763804.222041:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763804.222043:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763804.222046:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58405:58405:256:4294967295] 192.168.203.110@tcp LPNI seq info [58405:58405:8:4294967295]
00000400:00000200:1.0:1787763804.222048:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763804.222053:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763804.222056:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763804.222059:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763804.222061:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763804.222063:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787763804.222066:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763804.222067:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787763804.222070:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763804.222072:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763804.222074:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763804.222076:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763804.222078:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763804.222079:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763804.222080:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763804.222083:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832661760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763829 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.222088:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763804.222090:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832661760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763829 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.222094:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763804.222095:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763804.222097:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763804.222099:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763804.222101:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763804.222102:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316191269
00000100:00000001:1.0:1787763804.222104:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316192499
00000100:00000001:1.0:1787763804.222106:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316191269
00000100:00000001:1.0:1787763804.222107:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316192499
00000100:00000001:1.0:1787763804.222108:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763804.222110:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763804.222111:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763804.222114:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763804/real 1787763804]  req@ffff9751f865ad00 x1874599832661760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763804.222119:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763804.222120:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763804.222121:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763804.222122:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763804.222123:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763804.222124:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763804.222126:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832661760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763804.222131:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832661760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.222161:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763804.222162:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763804.222165:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763804.222166:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763804.222173:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832661760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.222179:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763804.222187:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832661760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.222192:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763804.222194:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763804.222195:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763804.222198:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763804.222200:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763804.222203:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763804.222205:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763804.222208:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763804.222209:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763804.222212:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832661760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763804.222218:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832661760:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763804.222222:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763804.222224:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763804.222226:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832661504/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.222231:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763804.222233:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763804.222235:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb800.
02000000:00000001:1.0:1787763804.222238:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763804.222240:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763804.222241:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763804.222242:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763804.222244:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:1.0:1787763804.222249:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763804.222251:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763804.222253:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:1.0:1787763804.222258:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763804.222261:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763804.222263:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763804.222268:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832661632/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.222273:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763804.222274:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763804.222276:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763804.222278:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763804.222278:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763804.222279:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763804.222280:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763804.222282:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:1.0:1787763804.222283:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763804.222284:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763804.222284:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787763804.222286:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763804.222287:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763804.222287:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763804.222289:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832661760/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763829 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763804.222293:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763804.222294:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763804.222294:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763804.222295:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763804.222296:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763804.222297:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763804.222298:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763804.222299:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:1.0:1787763804.222300:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763804.222301:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763804.222301:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:1.0:1787763804.222303:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763804.222304:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763804.222305:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763804.222307:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763804.222308:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763804.222309:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763804.222310:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763804.860187:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763804.860194:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763804.860197:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763804.860202:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763804.860206:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763804.860209:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763804.860211:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763804.860214:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763804.860217:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763804.860221:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763804.860226:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763804.860233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763804.860236:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763804.860241:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763804.860243:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763804.860265:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763804.860269:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763804.860273:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763805.244232:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763805.244239:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763805.244241:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763805.244246:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763805.244251:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763805.244254:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763805.244256:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763805.244258:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763805.244260:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763805.500210:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316192499, at 4316192512
00000100:00000010:3.0:1787763805.500228:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4900.
00000100:00000040:2.0:1787763805.500261:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316192499, at 4316192512
00000100:00000010:2.0:1787763805.500269:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c48a0.
00000100:00000001:2.0:1787763805.884201:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763805.884208:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763805.884210:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763805.884215:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763805.884217:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763805.884219:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763805.884222:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763805.884224:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763805.884227:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763805.884229:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763805.884230:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763805.884233:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763805.884234:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763805.884236:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763805.884241:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763805.884243:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763805.884245:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763805.884247:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763805.884249:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763805.884252:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763805.884253:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763805.884299:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763805.884303:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763805.884306:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763806.268202:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763806.268208:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763806.268211:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763806.268216:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763806.268220:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763806.268224:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763806.268227:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763806.268229:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763806.268233:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763806.908255:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763806.908265:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763806.908265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763806.908275:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763806.908278:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763806.908282:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763806.908284:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763806.908295:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763806.908296:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763806.908301:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763806.908304:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763806.908310:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763806.908314:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763806.908317:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763806.908322:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763806.908391:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763806.908398:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763806.908403:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763807.292171:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763807.292177:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763807.292179:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763807.292182:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763807.292185:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763807.292188:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763807.292190:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763807.292192:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763807.292194:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763807.932262:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763807.932273:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763807.932277:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763807.932285:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763807.932291:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763807.932298:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763807.932302:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763807.932306:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763807.932310:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763807.932326:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763807.932334:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763807.932341:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763807.932344:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763807.932349:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763807.932350:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763807.932416:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763807.932423:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763807.932428:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763808.316255:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763808.316264:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763808.316267:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763808.316273:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763808.316277:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763808.316281:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763808.316284:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763808.316287:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763808.316290:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763808.956200:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763808.956208:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763808.956210:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763808.956214:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763808.956215:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763808.956218:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763808.956220:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763808.956223:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763808.956226:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763808.956227:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763808.956229:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763808.956230:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763808.956233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763808.956234:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763808.956235:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763808.956281:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763808.956285:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763808.956287:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787763809.340261:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763809.340276:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787763809.340293:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763809.340298:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763809.340306:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763809.340317:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21523
00000400:00000200:2.0:1787763809.340327:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763809.340335:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763809.340339:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763809.340344:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763809.340348:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763809.340351:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763809.340354:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763809.340360:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763809.340363:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787763809.340368:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763809.340372:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787763809.340376:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787763809.340381:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787763809.340385:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763809.340388:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763809.340391:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763809.340395:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763809.340401:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787763809.340409:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763809.340415:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763809.340416:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
02000000:00000001:2.0:1787763809.340421:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:0.0:1787763809.340424:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763809.340429:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763809.340432:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:0.0:1787763809.340435:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763809.340445:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
02000000:00000001:2.0:1787763809.340447:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763809.340450:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763809.340457:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763809.340458:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763809.340465:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763809.340467:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763809.340471:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763809.340486:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658780.
00000020:00000040:2.0:1787763809.340496:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787763809.340503:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763809.340506:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763809.340510:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763809.340514:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763809.340520:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763809.340532:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5b8c00.
00000100:00000001:2.0:1787763809.340541:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763809.340580:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658780 x1874599832661888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763809.340595:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763809.340600:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763809.340606:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000744875b5] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658780 x1874599832661888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763809.340635:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763809.340639:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763809.340643:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787763809.340645:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763809.340648:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763809.340652:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763809.340655:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21523
00000400:00000200:2.0:1787763809.340662:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763809.340668:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763809.340672:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763809.340677:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763809.340680:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763809.340681:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787763809.340691:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1787763809.340691:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763809.340699:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1787763809.340700:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763809.340707:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763809.340710:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:2.0:1787763809.340713:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763809.340718:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000040:1.0:1787763809.340719:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832661888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787763809.340722:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:2.0:1787763809.340734:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:1.0:1787763809.340748:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:2.0:1787763809.340752:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763809.340753:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763809.340761:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1787763809.340765:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1787763809.340768:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763809.340775:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763809.340777:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00100000:1.0:1787763809.340785:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832661888:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787763809.340789:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787763809.340795:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:2.0:1787763809.340802:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:1.0:1787763809.340803:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:2.0:1787763809.340815:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:1.0:1787763809.340822:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763809.340827:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763809.340828:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1787763809.340836:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:1.0:1787763809.340840:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca400.
02000000:00000001:2.0:1787763809.340845:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763809.340847:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763809.340857:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763809.340864:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:1.0:1787763809.340865:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000008:00100000:2.0:1787763809.340871:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000010:1.0:1787763809.340874:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000008:00000001:2.0:1787763809.340878:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763809.340883:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763809.340887:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000010:2.0:1787763809.340891:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000020:00000040:2.0:1787763809.340899:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000100:00000200:1.0:1787763809.340900:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832661888, portal 25
02000000:00000001:2.0:1787763809.340906:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1787763809.340907:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:2.0:1787763809.340913:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763809.340914:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000001:2.0:1787763809.340924:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763809.340928:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:1.0:1787763809.340934:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832661888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763834 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787763809.340935:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763809.340947:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bbc00.
00000100:00000001:1.0:1787763809.340958:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1787763809.340960:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763809.340968:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763809.340975:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00100000:2.0:1787763809.340980:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832662016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:1.0:1787763809.340983:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832661888, offset 0
00000100:00000001:2.0:1787763809.341001:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1787763809.341002:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787763809.341012:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763809.341022:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832662016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787763809.341023:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763809.341041:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:2.0:1787763809.341044:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763809.341050:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:2.0:1787763809.341052:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787763809.341061:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00080000:2.0:1787763809.341062:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:1.0:1787763809.341070:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1787763809.341071:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1787763809.341077:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00080000:2.0:1787763809.341081:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1787763809.341086:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58406:58406:256:4294967295] 192.168.203.110@tcp LPNI seq info [58406:58406:8:4294967295]
00000100:00000001:2.0:1787763809.341093:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1787763809.341098:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:2.0:1787763809.341108:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21523
00000400:00000200:1.0:1787763809.341122:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763809.341131:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763809.341174:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000400:00000200:2.0:1787763809.341192:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:1.0:1787763809.341203:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000400:00000200:2.0:1787763809.341203:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763809.341212:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000200:1.0:1787763809.341218:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000100:00000040:2.0:1787763809.341220:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763809.341229:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763809.341233:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763809.341236:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763809.341241:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763809.341244:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787763809.341247:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763809.341255:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000200:1.1:1787763809.341255:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:2.0:1787763809.341266:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:1.1:1787763809.341269:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000020:00000001:2.0:1787763809.341275:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000100:1.0:1787763809.341284:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000040:2.0:1787763809.341287:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000010:1.0:1787763809.341291:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000100:00000001:2.0:1787763809.341293:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763809.341302:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:1.0:1787763809.341305:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1787763809.341307:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763809.341315:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:1.0:1787763809.341323:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:2.0:1787763809.341329:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:1.0:1787763809.341337:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000100:00000001:2.0:1787763809.341337:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763809.341346:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
02000000:00000001:2.0:1787763809.341347:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763809.341356:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763809.341358:0:257575:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:2.0:1787763809.341366:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787763809.341373:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832661888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763834 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763809.341374:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787763809.341380:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787763809.341391:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763809.341403:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763809.341406:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763809.341416:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832661888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763834 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1787763809.341419:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000020:00000040:2.0:1787763809.341432:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000001:1.0:1787763809.341436:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763809.341442:0:257575:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:2.0:1787763809.341444:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1787763809.341451:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763809.341451:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763809.341462:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:1.0:1787763809.341468:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:2.0:1787763809.341471:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1787763809.341475:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316197620
02000000:00000001:2.0:1787763809.341480:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763809.341487:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4780.
02000000:00000010:2.0:1787763809.341494:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5ba000.
00000100:00000001:1.0:1787763809.341500:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787763809.341503:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763809.341513:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316197620
00000100:00100000:2.0:1787763809.341519:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832662144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:1.0:1787763809.341524:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4360.
00000100:00000001:1.0:1787763809.341532:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763809.341540:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763809.341542:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763809.341570:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763809.341574:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763809.341577:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000040:2.0:1787763809.341591:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832662144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763809.341594:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763809/real 1787763809]  req@ffff9751f8658780 x1874599832661888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763809.341636:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1787763809.341641:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:2.0:1787763809.341643:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000001:1.0:1787763809.341648:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:2.0:1787763809.341653:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000010:1.0:1787763809.341655:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000100:00080000:2.0:1787763809.341662:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787763809.341664:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763809.341673:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00080000:2.0:1787763809.341674:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787763809.341681:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1787763809.341682:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00080000:2.0:1787763809.341689:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:1.0:1787763809.341692:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832661888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763809.341695:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763809.341705:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21523
00000400:00000200:2.0:1787763809.341718:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000200:1.0:1787763809.341724:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832661888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787763809.341733:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763809.341745:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787763809.341747:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763809.341752:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763809.341758:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:2.0:1787763809.341761:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763809.341762:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763809.341774:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:1.0:1787763809.341776:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832661888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787763809.341790:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763809.341794:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763809.341803:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:1.0:1787763809.341810:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832661888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787763809.341813:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763809.341821:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787763809.341829:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:1.0:1787763809.341831:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000040:2.0:1787763809.341837:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000001:1.0:1787763809.341839:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1787763809.341846:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:1.0:1787763809.341848:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763809.341854:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1787763809.341855:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:1.0:1787763809.341860:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763809.341868:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:2.0:1787763809.341868:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763809.341877:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00080000:1.0:1787763809.341880:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000040:2.0:1787763809.341886:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763809.341891:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00080000:1.0:1787763809.341894:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000020:00000040:2.0:1787763809.341897:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:1.0:1787763809.341901:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00080000:2.0:1787763809.341907:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000040:1.0:1787763809.341915:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832661888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763809.341917:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763809.341925:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763809.341933:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:1.0:1787763809.341942:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832661888:192.168.203.110@tcp:250:kworker.0
02000000:00000001:2.0:1787763809.341943:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763809.341951:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763809.341956:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000008:00100000:2.0:1787763809.341958:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1787763809.341963:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000008:00000001:2.0:1787763809.341966:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763809.341974:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763809.341975:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832661888/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1787763809.341981:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
00000020:00000040:2.0:1787763809.341989:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:1.0:1787763809.341993:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763809.341997:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763809.341999:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000001:2.0:1787763809.342003:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763809.342006:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca400.
00000100:00000001:2.0:1787763809.342008:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763809.342013:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
02000000:00000001:2.0:1787763809.342014:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000020:00000001:1.0:1787763809.342021:0:257575:0:(genops.c:1217:class_import_put()) Process entered
02000000:00000001:2.0:1787763809.342023:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1787763809.342026:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
02000000:00000010:2.0:1787763809.342032:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bb800.
00000020:00000001:1.0:1787763809.342033:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763809.342040:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
00000100:00000001:2.0:1787763809.342043:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763809.342047:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763809.342052:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763809.342056:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00100000:2.0:1787763809.342058:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832662272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763809.342062:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763809.342067:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763809.342076:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763809.342089:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763809.342092:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763809.342094:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763809.342100:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763809.342103:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763809.342104:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763809.342109:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763809.342131:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832662016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787763809.342173:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832662272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.342181:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763809.342189:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763809.342201:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1787763809.342201:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763809.342208:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1787763809.342213:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21534)
02000000:00000001:1.0:1787763809.342221:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763809.342233:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832662016:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763809.342241:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763809.342246:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763809.342249:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763809.342251:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763809.342256:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763809.342259:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763809.342265:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763809.342268:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763809.342273:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763809.342278:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832662016, portal 10
00000100:00000001:1.0:1787763809.342280:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763809.342282:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:1.0:1787763809.342292:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832662016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763834 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.342301:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763809.342304:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763809.342307:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763809.342310:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832662016, offset 0
00000400:00000200:1.0:1787763809.342315:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763809.342321:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763809.342326:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763809.342331:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763809.342334:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763809.342338:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763809.342341:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763809.342345:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58407:58407:256:4294967295] 192.168.203.110@tcp LPNI seq info [58407:58407:8:4294967295]
00000400:00000200:1.0:1787763809.342351:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763809.342360:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763809.342366:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763809.342371:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763809.342376:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763809.342380:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:1.1:1787763809.342386:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763809.342390:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000800:00000100:1.0:1787763809.342395:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763809.342398:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763809.342402:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763809.342407:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763809.342411:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763809.342413:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763809.342416:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763809.342421:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832662016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763834 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.342431:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763809.342435:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832662016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763834 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.342444:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763809.342446:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763809.342450:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763809.342455:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763809.342459:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763809.342461:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316196385
00000100:00000001:1.0:1787763809.342465:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316197620
00000100:00000001:1.0:1787763809.342468:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316196385
00000100:00000001:1.0:1787763809.342471:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316197620
00000100:00000001:1.0:1787763809.342474:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763809.342476:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763809.342479:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763809.342499:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763809/real 1787763809]  req@ffff9751f865ad00 x1874599832662016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763809.342524:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763809.342526:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763809.342528:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763809.342531:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763809.342533:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763809.342536:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763809.342540:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832662016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763809.342572:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832662016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.342581:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763809.342583:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763809.342585:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763809.342587:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763809.342592:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832662016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.342600:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763809.342606:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832662016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.342615:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763809.342618:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763809.342620:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763809.342621:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763809.342624:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763809.342637:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763809.342640:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763809.342644:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763809.342646:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763809.342652:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832662016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763809.342662:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832662016:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763809.342667:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763809.342671:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832662144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.342679:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763809.342681:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763809.342683:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763809.342686:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763809.342689:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763809.342692:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832662144:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763809.342696:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763809.342699:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763809.342701:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763809.342703:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763809.342708:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763809.342711:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763809.342715:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763809.342719:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763809.342723:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763809.342727:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832662144, portal 10
00000100:00000001:1.0:1787763809.342729:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763809.342731:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787763809.342739:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832662144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763834 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.342747:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763809.342751:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763809.342753:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763809.342756:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832662144, offset 0
00000400:00000200:1.0:1787763809.342761:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763809.342767:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763809.342771:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763809.342776:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763809.342779:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763809.342783:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763809.342786:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763809.342790:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58408:58408:256:4294967295] 192.168.203.110@tcp LPNI seq info [58408:58408:8:4294967295]
00000400:00000200:1.0:1787763809.342795:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763809.342804:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763809.342809:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763809.342813:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763809.342818:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763809.342821:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:1.1:1787763809.342827:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763809.342831:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000800:00000100:1.0:1787763809.342835:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763809.342838:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763809.342842:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763809.342846:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763809.342850:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763809.342853:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763809.342856:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763809.342860:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832662144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763834 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.342870:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763809.342874:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832662144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763834 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.342881:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763809.342883:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763809.342887:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763809.342890:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763809.342894:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763809.342897:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316196373
00000100:00000001:1.0:1787763809.342900:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316197620
00000100:00000001:1.0:1787763809.342903:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316196373
00000100:00000001:1.0:1787763809.342906:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316197620
00000100:00000001:1.0:1787763809.342908:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763809.342911:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763809.342913:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763809.342917:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763809/real 1787763809]  req@ffff9751f8659a40 x1874599832662144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763809.342927:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763809.342929:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763809.342931:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763809.342934:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763809.342936:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763809.342938:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763809.342942:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832662144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763809.342952:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832662144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.342959:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763809.342961:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763809.342963:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763809.342965:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763809.342969:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832662144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.342977:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763809.342982:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832662144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.342990:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763809.342994:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763809.342996:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763809.342998:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763809.343000:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763809.343002:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763809.343006:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763809.343009:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763809.343012:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763809.343016:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832662144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763809.343025:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832662144:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763809.343030:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763809.343032:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763809.343036:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832662016/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.343044:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763809.343046:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763809.343048:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763809.343051:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763809.343053:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763809.343055:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763809.343057:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763809.343059:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:1.0:1787763809.343062:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763809.343064:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763809.343066:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:1.0:1787763809.343070:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763809.343071:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763809.343073:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763809.343077:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832662144/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.343093:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763809.343094:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763809.343097:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763809.343099:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763809.343100:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763809.343102:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763809.343104:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763809.343106:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:1.0:1787763809.343108:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763809.343109:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763809.343111:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787763809.343114:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763809.343115:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763809.343117:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763809.343121:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763809.343123:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763809.343126:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763809.343127:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763809.343129:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763809.343168:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832662272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.343183:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763809.343186:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763809.343191:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763809.343194:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763809.343198:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763809.343205:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832662272:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763809.343212:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763809.343217:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763809.343220:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763809.343224:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763809.343231:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763809.343235:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763809.343243:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763809.343249:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763809.343254:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763809.343258:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832662272, portal 4
00000100:00000001:1.0:1787763809.343261:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763809.343263:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787763809.343271:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832662272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763834 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.343280:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763809.343283:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763809.343286:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763809.343288:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832662272, offset 0
00000400:00000200:1.0:1787763809.343293:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763809.343299:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763809.343304:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763809.343308:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763809.343312:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763809.343315:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763809.343318:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763809.343322:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58409:58409:256:4294967295] 192.168.203.110@tcp LPNI seq info [58409:58409:8:4294967295]
00000400:00000200:1.0:1787763809.343327:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763809.343335:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763809.343341:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763809.343345:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763809.343350:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763809.343354:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000800:00000200:1.1:1787763809.343359:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763809.343363:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 11 secs later)
00000800:00000100:1.0:1787763809.343367:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763809.343370:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763809.343374:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763809.343378:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763809.343382:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763809.343385:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763809.343388:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763809.343393:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832662272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763834 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.343403:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763809.343407:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832662272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763834 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.343415:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763809.343417:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763809.343421:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763809.343424:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763809.343428:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763809.343430:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316196391
00000100:00000001:1.0:1787763809.343433:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316197620
00000100:00000001:1.0:1787763809.343436:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316196391
00000100:00000001:1.0:1787763809.343439:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316197620
00000100:00000001:1.0:1787763809.343442:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763809.343445:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763809.343447:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763809.343451:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763809/real 1787763809]  req@ffff9751f8658000 x1874599832662272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763809.343460:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763809.343462:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763809.343464:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763809.343466:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763809.343468:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763809.343470:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763809.343474:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832662272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763809.343484:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832662272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.343492:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763809.343494:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763809.343497:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763809.343499:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763809.343503:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832662272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.343511:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763809.343515:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832662272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.343523:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763809.343526:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763809.343528:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763809.343530:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763809.343532:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763809.343534:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763809.343537:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763809.343541:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763809.343571:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763809.343579:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832662272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763809.343589:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832662272:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763809.343594:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763809.343596:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763809.343600:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832662272/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763834 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763809.343608:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763809.343609:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763809.343613:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763809.343616:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763809.343618:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763809.343620:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763809.343622:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763809.343625:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:1.0:1787763809.343627:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763809.343629:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763809.343631:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:1.0:1787763809.343634:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763809.343636:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763809.343638:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763809.343643:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763809.343645:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763809.343647:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763809.343649:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763809.980263:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763809.980275:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763809.980278:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763809.980286:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763809.980292:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763809.980299:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763809.980303:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763809.980307:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763809.980311:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763809.980329:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763809.980337:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763809.980343:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763809.980345:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763809.980349:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763809.980351:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763809.980480:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763809.980487:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763809.980492:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763810.364228:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763810.364236:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763810.364237:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763810.364242:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763810.364245:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763810.364248:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763810.364251:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763810.364252:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763810.364255:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763810.620222:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316197620, at 4316197632
00000100:00000010:3.0:1787763810.620235:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4360.
00000100:00000040:2.0:1787763810.620258:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316197620, at 4316197632
00000100:00000010:2.0:1787763810.620266:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4780.
00000100:00000001:2.0:1787763811.004204:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763811.004211:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763811.004214:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763811.004219:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763811.004224:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763811.004227:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763811.004230:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763811.004232:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763811.004235:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763811.004235:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763811.004240:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763811.004244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763811.004246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763811.004248:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763811.004250:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763811.004286:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763811.004290:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763811.004293:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763811.388241:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763811.388248:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763811.388250:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763811.388254:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763811.388257:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763811.388259:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763811.388262:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763811.388264:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763811.388266:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763812.028284:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763812.028292:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763812.028295:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763812.028300:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763812.028302:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763812.028313:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763812.028314:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763812.028319:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763812.028323:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763812.028328:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763812.028330:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763812.028336:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763812.028338:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763812.028342:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787763812.028348:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763812.028349:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763812.028359:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787763812.028364:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763812.028364:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763812.028372:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763812.028377:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763812.028451:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763812.028459:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763812.028464:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763812.412280:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763812.412290:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763812.412293:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763812.412302:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763812.412309:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763812.412313:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763812.412317:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763812.412320:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763812.412324:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763813.052245:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763813.052256:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763813.052259:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763813.052267:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763813.052274:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763813.052279:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763813.052283:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763813.052287:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763813.052292:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763813.052311:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763813.052319:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763813.052327:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763813.052329:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763813.052334:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763813.052336:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763813.052491:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763813.052500:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763813.052506:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763813.436250:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763813.436258:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763813.436260:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763813.436266:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763813.436270:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763813.436274:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763813.436277:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763813.436280:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763813.436282:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763814.076319:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763814.076331:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763814.076334:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763814.076339:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763814.076345:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763814.076350:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763814.076355:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763814.076360:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763814.076365:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763814.076366:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763814.076370:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763814.076371:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763814.076395:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763814.076410:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000020:00000001:1.0:1787763814.076414:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000100:00000001:2.0:1787763814.076415:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763814.076422:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763814.076426:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00010000:1.0:1787763814.076429:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763814.076434:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763814.076441:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763814.076445:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763814.076449:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763814.076452:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763814.460288:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763814.460301:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787763814.460306:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763814.460311:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763814.460319:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763814.460330:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21529
00000100:00000001:0.0:1787763814.460333:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763814.460344:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000400:00000200:2.0:1787763814.460346:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763814.460348:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763814.460355:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000400:00000200:2.0:1787763814.460358:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:0.0:1787763814.460362:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:2.0:1787763814.460364:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787763814.460369:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763814.460373:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763814.460376:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763814.460384:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787763814.460385:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763814.460393:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763814.460394:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763814.460406:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763814.460412:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763814.460416:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787763814.460419:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763814.460424:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787763814.460427:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787763814.460431:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787763814.460436:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763814.460440:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763814.460444:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763814.460448:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763814.460478:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787763814.460482:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763814.460486:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763814.460489:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763814.460495:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763814.460498:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763814.460501:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787763814.460504:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763814.460531:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000020:00000040:2.0:1787763814.460536:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787763814.460540:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763814.460542:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763814.460545:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763814.460548:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763814.460553:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763814.460562:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bb800.
00000100:00000001:2.0:1787763814.460570:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763814.460581:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832662400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763814.460593:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763814.460597:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763814.460602:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832662400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763814.460636:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763814.460640:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763814.460645:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787763814.460647:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763814.460650:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763814.460654:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763814.460658:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21529
00000400:00000200:2.0:1787763814.460663:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763814.460665:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:2.0:1787763814.460669:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1787763814.460671:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763814.460676:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00080000:2.0:1787763814.460677:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763814.460687:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832662400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787763814.460687:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763814.460695:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787763814.460702:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:2.0:1787763814.460702:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763814.460705:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763814.460711:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1787763814.460711:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:1.0:1787763814.460715:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:2.0:1787763814.460717:0:304484:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:1.0:1787763814.460719:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1787763814.460722:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00100000:1.0:1787763814.460725:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832662400:192.168.203.110@tcp:250:kworker.0
00000020:00000001:2.0:1787763814.460729:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:1.0:1787763814.460731:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:2.0:1787763814.460733:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
02000000:00000001:1.0:1787763814.460736:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000001:2.0:1787763814.460737:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:1.0:1787763814.460740:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763814.460741:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:1.0:1787763814.460745:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:2.0:1787763814.460749:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763814.460752:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000010:1.0:1787763814.460753:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca800.
00000100:00000040:2.0:1787763814.460757:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:1.0:1787763814.460759:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763814.460762:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763814.460766:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000400:00000200:1.0:1787763814.460769:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:2.0:1787763814.460773:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000010:1.0:1787763814.460775:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000100:00000001:2.0:1787763814.460777:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763814.460781:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:1.0:1787763814.460783:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
02000000:00000001:2.0:1787763814.460787:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:1.0:1787763814.460791:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832662400, portal 25
02000000:00000001:2.0:1787763814.460792:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763814.460796:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763814.460797:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763814.460799:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000008:00100000:2.0:1787763814.460803:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000040:1.0:1787763814.460809:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832662400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763839 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:2.0:1787763814.460809:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763814.460814:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763814.460823:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000010:2.0:1787763814.460823:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
00000020:00000040:2.0:1787763814.460828:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000100:00000040:1.0:1787763814.460829:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
02000000:00000001:2.0:1787763814.460835:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:1.0:1787763814.460836:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
02000000:00000001:2.0:1787763814.460837:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787763814.460842:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832662400, offset 0
00000100:00000001:2.0:1787763814.460842:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763814.460846:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763814.460850:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763814.460854:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
02000000:00000010:2.0:1787763814.460858:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5ba000.
00000400:00000200:1.0:1787763814.460864:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763814.460865:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763814.460874:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00100000:2.0:1787763814.460874:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832662528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787763814.460883:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1787763814.460890:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1787763814.460891:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:2.0:1787763814.460896:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763814.460899:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000040:2.0:1787763814.460905:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832662528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787763814.460906:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763814.460936:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58410:58410:256:4294967295] 192.168.203.110@tcp LPNI seq info [58410:58410:8:4294967295]
00000100:00000001:2.0:1787763814.460948:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763814.460958:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:2.0:1787763814.460958:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763814.460970:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1787763814.460973:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763814.460977:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763814.460988:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763814.460997:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21529
00000400:00000200:1.0:1787763814.461001:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763814.461019:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763814.461026:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763814.461035:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000800:00000010:1.0:1787763814.461036:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000100:00080000:2.0:1787763814.461039:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763814.461048:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763814.461051:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.0:1787763814.461059:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:2.0:1787763814.461060:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763814.461069:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:1.0:1787763814.461071:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000020:00000001:2.0:1787763814.461078:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763814.461082:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787763814.461089:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:1.1:1787763814.461090:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:2.0:1787763814.461093:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000200:1.1:1787763814.461101:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000020:00000001:2.0:1787763814.461102:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:2.0:1787763814.461113:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000100:1.0:1787763814.461119:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763814.461128:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000100:00000040:2.0:1787763814.461128:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:1.0:1787763814.461140:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1787763814.461169:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763814.461175:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763814.461178:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763814.461182:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:2.0:1787763814.461187:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763814.461190:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763814.461198:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:2.0:1787763814.461199:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763814.461208:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1787763814.461211:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
02000000:00000001:2.0:1787763814.461214:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763814.461217:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:2.0:1787763814.461220:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787763814.461226:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1787763814.461228:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000008:00000001:2.0:1787763814.461234:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787763814.461242:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832662400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763839 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763814.461243:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763814.461256:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000020:00000040:2.0:1787763814.461265:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000001:1.0:1787763814.461270:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1787763814.461273:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763814.461279:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763814.461281:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832662400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763839 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763814.461285:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763814.461290:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763814.461296:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763814.461306:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763814.461309:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bbc00.
00000100:00000001:1.0:1787763814.461312:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763814.461322:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763814.461325:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763814.461335:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832662656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787763814.461336:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763814.461345:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316202740
00000100:00000010:1.0:1787763814.461358:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4600.
00000100:00000001:2.0:1787763814.461359:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763814.461368:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763814.461374:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000040:2.0:1787763814.461376:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832662656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.461388:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316202740
00000100:00000001:2.0:1787763814.461391:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763814.461396:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4ea0.
00000100:00000040:2.0:1787763814.461398:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787763814.461404:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000040:2.0:1787763814.461407:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:1.0:1787763814.461412:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:2.0:1787763814.461414:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787763814.461420:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763814.461423:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787763814.461426:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1787763814.461428:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763814.461434:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:1.0:1787763814.461440:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763814/real 1787763814]  req@ffff9751f86583c0 x1874599832662400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763814.461442:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763814.461469:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21529
00000400:00000200:2.0:1787763814.461480:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000001:1.0:1787763814.461482:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000200:2.0:1787763814.461485:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000001:1.0:1787763814.461486:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763814.461491:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000100:00080000:2.0:1787763814.461492:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1787763814.461496:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000100:00000040:2.0:1787763814.461498:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000010:1.0:1787763814.461500:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:2.0:1787763814.461502:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787763814.461505:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:2.0:1787763814.461508:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763814.461527:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000200:1.0:1787763814.461529:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832662400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787763814.461530:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763814.461534:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787763814.461537:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763814.461541:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787763814.461546:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000200:1.0:1787763814.461548:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832662400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787763814.461550:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:2.0:1787763814.461558:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763814.461562:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787763814.461566:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000040:2.0:1787763814.461567:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763814.461570:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763814.461571:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787763814.461574:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000040:2.0:1787763814.461575:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:1.0:1787763814.461577:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1787763814.461581:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00100000:1.0:1787763814.461585:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832662400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763814.461586:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763814.461592:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763814.461596:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763814.461599:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1787763814.461600:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763814.461605:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:1.0:1787763814.461608:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832662400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:2.0:1787763814.461609:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1787763814.461616:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763814.461621:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763814.461627:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000010:2.0:1787763814.461628:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000100:00000001:1.0:1787763814.461633:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:2.0:1787763814.461635:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:1.0:1787763814.461637:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:2.0:1787763814.461639:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1787763814.461640:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1787763814.461643:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763814.461645:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763814.461647:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1787763814.461648:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:2.0:1787763814.461650:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00080000:1.0:1787763814.461653:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000001:2.0:1787763814.461654:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763814.461662:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
02000000:00000010:2.0:1787763814.461662:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5b8c00.
00000100:00000001:1.0:1787763814.461668:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:2.0:1787763814.461668:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763814.461675:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832662400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763814.461680:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832662784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:1.0:1787763814.461695:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832662400:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787763814.461698:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763814.461702:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763814.461702:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763814.461707:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763814.461712:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832662784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787763814.461714:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832662400/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763814.461744:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763814.461752:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21539)
00000100:00000001:1.0:1787763814.461754:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763814.461757:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763814.461760:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763814.461763:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763814.461764:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763814.461766:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763814.461768:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763814.461771:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:1.0:1787763814.461774:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763814.461775:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763814.461776:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:1.0:1787763814.461778:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763814.461779:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763814.461781:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763814.461787:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763814.461789:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763814.461791:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763814.461792:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763814.461793:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763814.461796:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832662528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.461802:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763814.461803:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763814.461805:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763814.461807:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763814.461809:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763814.461811:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832662528:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763814.461814:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763814.461815:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763814.461816:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763814.461818:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763814.461820:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763814.461822:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763814.461825:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763814.461828:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763814.461831:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763814.461834:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832662528, portal 10
00000100:00000001:1.0:1787763814.461835:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763814.461836:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787763814.461842:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832662528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763839 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.461847:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763814.461849:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763814.461850:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763814.461852:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832662528, offset 0
00000400:00000200:1.0:1787763814.461855:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763814.461858:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763814.461862:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763814.461864:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763814.461867:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763814.461869:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763814.461871:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763814.461873:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58411:58411:256:4294967295] 192.168.203.110@tcp LPNI seq info [58411:58411:8:4294967295]
00000400:00000200:1.0:1787763814.461876:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763814.461882:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763814.461886:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763814.461889:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763814.461910:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763814.461913:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1787763814.461919:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763814.461922:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:1.0:1787763814.461927:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763814.461930:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763814.461934:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763814.461938:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763814.461943:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763814.461945:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763814.461950:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763814.461957:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832662528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763839 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.461974:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763814.461980:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832662528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763839 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.461993:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763814.461996:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763814.462003:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763814.462010:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763814.462017:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763814.462022:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316201504
00000100:00000001:1.0:1787763814.462028:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316202740
00000100:00000001:1.0:1787763814.462032:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316201504
00000100:00000001:1.0:1787763814.462037:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316202740
00000100:00000001:1.0:1787763814.462041:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763814.462045:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763814.462049:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763814.462057:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763814/real 1787763814]  req@ffff9751f8658000 x1874599832662528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763814.462068:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763814.462070:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763814.462073:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763814.462076:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763814.462079:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763814.462082:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763814.462086:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832662528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763814.462096:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832662528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.462112:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763814.462114:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763814.462117:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763814.462119:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763814.462123:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832662528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.462131:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763814.462136:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832662528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.462175:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763814.462180:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763814.462182:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763814.462184:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763814.462187:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763814.462202:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763814.462205:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763814.462209:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763814.462212:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763814.462218:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832662528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763814.462228:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832662528:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763814.462234:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763814.462238:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832662656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.462247:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763814.462249:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763814.462253:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763814.462256:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763814.462258:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763814.462262:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832662656:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763814.462266:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763814.462269:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763814.462271:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763814.462273:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763814.462278:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763814.462281:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763814.462285:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763814.462289:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763814.462293:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763814.462297:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832662656, portal 10
00000100:00000001:1.0:1787763814.462299:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763814.462301:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787763814.462309:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832662656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763839 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.462332:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763814.462334:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763814.462336:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763814.462338:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832662656, offset 0
00000400:00000200:1.0:1787763814.462341:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763814.462344:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763814.462347:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763814.462349:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763814.462352:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763814.462354:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763814.462356:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763814.462358:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58412:58412:256:4294967295] 192.168.203.110@tcp LPNI seq info [58412:58412:8:4294967295]
00000400:00000200:1.0:1787763814.462361:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763814.462366:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763814.462370:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763814.462373:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763814.462376:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763814.462378:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1787763814.462381:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763814.462383:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:1.0:1787763814.462387:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763814.462389:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763814.462391:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763814.462394:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763814.462396:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763814.462398:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763814.462400:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763814.462402:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832662656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763839 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.462409:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763814.462411:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832662656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763839 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.462416:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763814.462417:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763814.462419:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763814.462421:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763814.462424:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763814.462426:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316201493
00000100:00000001:1.0:1787763814.462428:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316202740
00000100:00000001:1.0:1787763814.462430:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316201493
00000100:00000001:1.0:1787763814.462431:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316202740
00000100:00000001:1.0:1787763814.462433:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763814.462434:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763814.462436:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763814.462438:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763814/real 1787763814]  req@ffff9751f8659a40 x1874599832662656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763814.462445:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763814.462446:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763814.462447:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763814.462449:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763814.462450:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763814.462451:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763814.462453:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832662656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763814.462459:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832662656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.462464:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763814.462465:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763814.462467:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763814.462468:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763814.462470:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832662656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.462475:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763814.462478:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832662656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.462483:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763814.462484:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763814.462485:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763814.462486:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763814.462488:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763814.462489:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763814.462491:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763814.462493:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763814.462494:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763814.462498:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832662656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763814.462504:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832662656:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763814.462507:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763814.462525:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832662784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.462531:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763814.462532:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763814.462534:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763814.462536:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763814.462537:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763814.462539:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832662784:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763814.462542:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763814.462544:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763814.462545:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763814.462546:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763814.462550:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb800.
02000000:00000001:1.0:1787763814.462551:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763814.462554:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763814.462556:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763814.462558:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763814.462561:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832662784, portal 4
00000100:00000001:1.0:1787763814.462562:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763814.462564:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:1.0:1787763814.462569:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832662784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763839 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.462574:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763814.462575:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763814.462577:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763814.462579:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832662784, offset 0
00000400:00000200:1.0:1787763814.462583:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763814.462586:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763814.462588:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763814.462591:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763814.462593:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763814.462595:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763814.462597:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763814.462599:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58413:58413:256:4294967295] 192.168.203.110@tcp LPNI seq info [58413:58413:8:4294967295]
00000400:00000200:1.0:1787763814.462603:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763814.462608:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763814.462611:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763814.462613:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763814.462616:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763814.462618:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:1.1:1787763814.462621:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763814.462623:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:1.0:1787763814.462626:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763814.462628:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763814.462630:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763814.462633:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763814.462635:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763814.462637:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763814.462638:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763814.462641:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832662784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763839 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.462647:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763814.462649:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832662784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763839 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.462655:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763814.462656:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763814.462658:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763814.462660:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763814.462662:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763814.462665:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316201510
00000100:00000001:1.0:1787763814.462666:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316202740
00000100:00000001:1.0:1787763814.462668:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316201510
00000100:00000001:1.0:1787763814.462670:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316202740
00000100:00000001:1.0:1787763814.462671:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763814.462673:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763814.462674:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763814.462677:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763814/real 1787763814]  req@ffff9751f865ad00 x1874599832662784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763814.462682:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763814.462684:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763814.462685:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763814.462687:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763814.462688:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763814.462689:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763814.462692:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832662784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763814.462698:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832662784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.462703:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763814.462704:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763814.462705:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763814.462706:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763814.462708:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832662784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.462713:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763814.462716:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832662784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.462721:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763814.462723:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763814.462724:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763814.462725:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763814.462727:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763814.462728:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763814.462730:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763814.462732:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763814.462734:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763814.462736:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832662784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763814.462742:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832662784:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763814.462744:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763814.462746:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763814.462748:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832662528/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.462753:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763814.462754:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763814.462755:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763814.462757:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763814.462758:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763814.462759:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763814.462760:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763814.462762:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:1.0:1787763814.462764:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763814.462765:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763814.462766:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:1.0:1787763814.462768:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763814.462769:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763814.462770:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763814.462773:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832662656/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.462782:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763814.462783:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763814.462785:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763814.462786:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763814.462787:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763814.462788:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763814.462789:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763814.462790:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:1.0:1787763814.462791:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763814.462792:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763814.462793:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787763814.462795:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763814.462796:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763814.462797:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763814.462799:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832662784/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763839 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763814.462803:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763814.462804:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763814.462805:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb800.
02000000:00000001:1.0:1787763814.462807:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763814.462807:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763814.462808:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763814.462809:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763814.462810:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:1.0:1787763814.462812:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763814.462813:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763814.462814:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:1.0:1787763814.462815:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763814.462816:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763814.462818:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763814.462820:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763814.462821:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763814.462823:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763814.462824:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763815.100262:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763815.100270:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763815.100273:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763815.100277:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763815.100280:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763815.100290:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763815.100291:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763815.100295:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763815.100300:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763815.100303:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763815.100307:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763815.100308:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763815.100314:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763815.100318:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763815.100321:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763815.100438:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763815.100446:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763815.100450:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763815.484259:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763815.484286:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763815.484289:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763815.484296:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763815.484301:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763815.484306:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763815.484309:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763815.484312:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763815.484315:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763815.740250:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316202740, at 4316202752
00000100:00000010:3.0:1787763815.740285:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4ea0.
00000100:00000040:2.0:1787763815.740361:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316202740, at 4316202752
00000100:00000010:2.0:1787763815.740379:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4600.
00000100:00000001:2.0:1787763816.124249:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763816.124261:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763816.124265:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763816.124273:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763816.124279:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763816.124284:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763816.124288:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763816.124292:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763816.124294:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763816.124297:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763816.124301:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763816.124310:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763816.124315:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763816.124317:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763816.124322:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763816.124327:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763816.124330:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763816.124333:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763816.124336:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763816.124340:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763816.124342:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763816.124475:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763816.124482:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763816.124487:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763816.508217:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763816.508227:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763816.508231:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763816.508239:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763816.508245:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763816.508250:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763816.508253:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763816.508256:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763816.508261:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763817.148324:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763817.148347:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763817.148348:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763817.148357:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763817.148359:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763817.148363:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763817.148364:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763817.148373:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763817.148374:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763817.148379:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763817.148395:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763817.148400:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763817.148404:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763817.148408:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763817.148412:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763817.148446:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763817.148451:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763817.148459:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763817.532234:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763817.532241:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763817.532243:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763817.532247:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763817.532251:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763817.532254:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763817.532257:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763817.532259:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763817.532261:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763818.172308:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763818.172310:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763818.172319:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763818.172320:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763818.172328:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763818.172332:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763818.172335:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763818.172337:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763818.172345:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763818.172347:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763818.172353:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763818.172354:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763818.172364:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763818.172370:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763818.172374:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763818.172441:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763818.172447:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763818.172453:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763818.556244:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763818.556270:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763818.556275:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763818.556283:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763818.556290:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763818.556294:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763818.556298:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763818.556301:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763818.556305:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763819.196315:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763819.196318:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763819.196326:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763819.196327:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763819.196334:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763819.196338:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763819.196343:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763819.196344:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763819.196356:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763819.196358:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763819.196363:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763819.196364:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763819.196371:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763819.196374:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763819.196378:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763819.196461:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763819.196468:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763819.196473:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787763819.580248:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763819.580262:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787763819.580265:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763819.580271:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763819.580278:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763819.580288:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21534
00000400:00000200:2.0:1787763819.580302:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763819.580306:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000400:00000200:2.0:1787763819.580313:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:0.0:1787763819.580316:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:2.0:1787763819.580321:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000001:0.0:1787763819.580323:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763819.580333:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00100000:0.0:1787763819.580334:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763819.580343:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787763819.580345:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763819.580354:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763819.580355:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763819.580362:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763819.580363:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:0.0:1787763819.580367:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1787763819.580371:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:0.0:1787763819.580374:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1787763819.580376:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787763819.580383:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763819.580388:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787763819.580391:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787763819.580394:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787763819.580398:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763819.580401:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763819.580405:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763819.580409:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763819.580413:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787763819.580418:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763819.580421:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763819.580425:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763819.580429:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763819.580433:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763819.580436:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787763819.580439:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763819.580449:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658780.
00000020:00000040:2.0:1787763819.580453:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787763819.580458:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763819.580460:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763819.580464:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763819.580467:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763819.580472:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763819.580506:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5ba000.
00000100:00000001:2.0:1787763819.580515:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763819.580532:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658780 x1874599832662912/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763819.580551:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763819.580559:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763819.580570:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000744875b5] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658780 x1874599832662912/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763819.580609:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763819.580616:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763819.580620:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787763819.580622:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763819.580625:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763819.580629:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763819.580632:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21534
00000400:00000200:2.0:1787763819.580639:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763819.580644:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763819.580648:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763819.580652:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763819.580657:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763819.580660:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763819.580663:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763819.580666:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763819.580668:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:2.0:1787763819.580670:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:1.0:1787763819.580674:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1787763819.580675:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:2.0:1787763819.580679:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:2.0:1787763819.580685:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:1.0:1787763819.580686:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1787763819.580691:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763819.580695:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1787763819.580697:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763819.580704:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763819.580708:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763819.580714:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00000040:1.0:1787763819.580719:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832662912/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787763819.580723:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763819.580733:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763819.580741:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763819.580751:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763819.580756:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1787763819.580757:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763819.580765:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763819.580766:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787763819.580775:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1787763819.580778:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:2.0:1787763819.580782:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763819.580788:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763819.580788:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763819.580798:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
02000000:00000001:1.0:1787763819.580800:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1787763819.580805:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787763819.580811:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763819.580815:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763819.580816:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832662912:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787763819.580818:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763819.580825:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763819.580831:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763819.580836:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000010:2.0:1787763819.580839:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:1.0:1787763819.580846:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1787763819.580848:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763819.580855:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763819.580863:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832663040/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1787763819.580866:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1787763819.580880:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000010:1.0:1787763819.580881:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb800.
00000100:00000001:2.0:1787763819.580886:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763819.580893:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763819.580896:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832663040/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787763819.580912:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763819.580917:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763819.580923:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:1.0:1787763819.580925:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000100:00080000:2.0:1787763819.580928:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1787763819.580935:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763819.580938:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763819.580941:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:1.0:1787763819.580945:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00080000:2.0:1787763819.580946:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21534
00000400:00000200:2.0:1787763819.580958:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763819.580963:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000200:1.0:1787763819.580965:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832662912, portal 25
00000100:00080000:2.0:1787763819.580966:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787763819.580975:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763819.580979:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000040:2.0:1787763819.580990:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763819.580996:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763819.580998:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832662912/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763844 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787763819.581004:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763819.581013:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763819.581022:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763819.581026:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787763819.581032:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763819.581036:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000100:00000001:1.0:1787763819.581041:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000001:2.0:1787763819.581042:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:2.0:1787763819.581049:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787763819.581058:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000040:2.0:1787763819.581060:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763819.581066:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:1.0:1787763819.581068:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000040:2.0:1787763819.581071:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763819.581079:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000200:1.0:1787763819.581083:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832662912, offset 0
00000020:00000040:2.0:1787763819.581084:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:2.0:1787763819.581094:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763819.581099:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763819.581102:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763819.581107:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1787763819.581109:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
02000000:00000001:2.0:1787763819.581111:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763819.581118:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787763819.581121:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787763819.581124:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763819.581127:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763819.581166:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000020:00000040:2.0:1787763819.581169:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787763819.581172:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763819.581173:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763819.581176:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763819.581178:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763819.581181:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763819.581186:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5b9800.
00000400:00000200:1.0:1787763819.581188:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763819.581192:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763819.581203:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832663168/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787763819.581211:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:2.0:1787763819.581226:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1787763819.581230:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1787763819.581236:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763819.581246:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000040:2.0:1787763819.581256:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832663168/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787763819.581262:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763819.581278:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:2.0:1787763819.581292:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763819.581293:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58414:58414:256:4294967295] 192.168.203.110@tcp LPNI seq info [58414:58414:8:4294967295]
00000100:00000040:2.0:1787763819.581305:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787763819.581313:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:2.0:1787763819.581321:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:2.0:1787763819.581331:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763819.581342:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000200:1.0:1787763819.581350:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1787763819.581350:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763819.581357:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763819.581366:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:1.0:1787763819.581374:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00080000:2.0:1787763819.581375:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21534
00000400:00000200:2.0:1787763819.581388:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000010:1.0:1787763819.581392:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000400:00000200:2.0:1787763819.581394:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763819.581402:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763819.581409:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763819.581418:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763819.581423:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763819.581425:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763819.581428:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763819.581430:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787763819.581432:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:1.0:1787763819.581435:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:2.0:1787763819.581435:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787763819.581444:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:2.0:1787763819.581450:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000800:00000200:1.0:1787763819.581453:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000100:00000040:2.0:1787763819.581455:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763819.581462:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763819.581470:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763819.581498:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:1.1:1787763819.581505:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:2.0:1787763819.581506:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:2.0:1787763819.581522:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000800:00000200:1.1:1787763819.581522:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000100:00000001:2.0:1787763819.581537:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:1.0:1787763819.581541:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
02000000:00000001:2.0:1787763819.581545:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000010:1.0:1787763819.581550:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
02000000:00000001:2.0:1787763819.581557:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1787763819.581565:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1787763819.581567:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763819.581578:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:1.0:1787763819.581587:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000008:00100000:2.0:1787763819.581589:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1787763819.581599:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763819.581601:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000100:00000001:2.0:1787763819.581604:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763819.581611:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000010:2.0:1787763819.581614:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
00000020:00000040:2.0:1787763819.581622:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:1.0:1787763819.581623:0:257575:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:2.0:1787763819.581627:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763819.581632:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763819.581637:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:1.0:1787763819.581641:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832662912/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763844 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787763819.581641:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763819.581646:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763819.581657:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bbc00.
00000100:00000001:2.0:1787763819.581667:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763819.581684:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00100000:2.0:1787763819.581684:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832663296/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1787763819.581701:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832662912/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763844 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763819.581704:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763819.581711:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763819.581724:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832663296/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.581746:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763819.581749:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763819.581753:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:2.0:1787763819.581758:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21544)
00000100:00000001:1.0:1787763819.581770:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763819.581783:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763819.581789:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316207860
00000100:00000010:1.0:1787763819.581801:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4ba0.
00000100:00000001:1.0:1787763819.581812:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787763819.581825:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316207860
00000100:00000010:1.0:1787763819.581831:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4420.
00000100:00000001:1.0:1787763819.581837:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763819.581843:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763819.581849:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763819.581853:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763819.581864:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763819/real 1787763819]  req@ffff9751f8658780 x1874599832662912/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763819.581883:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763819.581887:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763819.581891:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763819.581896:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763819.581900:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763819.581905:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763819.581913:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832662912/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763819.581933:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832662912/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.581948:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763819.581951:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763819.581956:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763819.581960:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763819.581969:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832662912/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.581984:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763819.581994:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832662912/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.582010:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763819.582015:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763819.582018:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763819.582022:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763819.582027:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763819.582030:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763819.582036:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763819.582045:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763819.582050:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763819.582059:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832662912/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763819.582077:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832662912:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763819.582086:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763819.582091:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763819.582098:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832662912/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.582113:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763819.582117:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763819.582122:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb800.
02000000:00000001:1.0:1787763819.582128:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763819.582167:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763819.582172:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763819.582177:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763819.582182:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:1.0:1787763819.582189:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763819.582192:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763819.582195:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:1.0:1787763819.582201:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763819.582204:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763819.582208:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763819.582220:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763819.582223:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763819.582227:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763819.582231:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763819.582234:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763819.582242:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832663040/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.582258:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763819.582261:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763819.582266:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763819.582270:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763819.582274:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763819.582280:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832663040:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763819.582288:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763819.582292:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763819.582295:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763819.582299:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763819.582305:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb800.
02000000:00000001:1.0:1787763819.582309:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763819.582317:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763819.582323:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763819.582329:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763819.582335:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832663040, portal 10
00000100:00000001:1.0:1787763819.582339:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763819.582342:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:1.0:1787763819.582354:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832663040/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763844 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.582368:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763819.582373:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763819.582379:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763819.582384:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832663040, offset 0
00000400:00000200:1.0:1787763819.582392:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763819.582401:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763819.582409:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763819.582415:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763819.582421:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763819.582426:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763819.582431:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763819.582438:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58415:58415:256:4294967295] 192.168.203.110@tcp LPNI seq info [58415:58415:8:4294967295]
00000400:00000200:1.0:1787763819.582447:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763819.582460:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763819.582470:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763819.582502:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763819.582513:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763819.582519:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1787763819.582527:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763819.582532:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:1.0:1787763819.582540:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763819.582543:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763819.582550:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763819.582557:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763819.582563:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763819.582566:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763819.582572:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763819.582580:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832663040/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763844 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.582596:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763819.582602:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832663040/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763844 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.582615:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763819.582617:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763819.582624:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763819.582631:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763819.582637:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763819.582641:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316206625
00000100:00000001:1.0:1787763819.582646:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316207860
00000100:00000001:1.0:1787763819.582650:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316206625
00000100:00000001:1.0:1787763819.582654:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316207860
00000100:00000001:1.0:1787763819.582658:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763819.582663:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763819.582667:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763819.582674:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763819/real 1787763819]  req@ffff9751f865ad00 x1874599832663040/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763819.582689:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763819.582692:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763819.582696:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763819.582700:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763819.582704:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763819.582708:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763819.582715:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832663040/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763819.582732:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832663040/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.582746:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763819.582749:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763819.582752:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763819.582755:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763819.582763:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832663040/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.582778:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763819.582787:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832663040/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.582802:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763819.582807:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763819.582811:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763819.582814:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763819.582818:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763819.582835:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763819.582841:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763819.582850:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763819.582854:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763819.582863:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832663040/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763819.582880:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832663040:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763819.582889:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763819.582896:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832663168/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.582910:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763819.582913:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763819.582917:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763819.582921:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763819.582925:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763819.582931:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832663168:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763819.582938:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763819.582942:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763819.582945:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763819.582949:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763819.582957:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763819.582961:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763819.582968:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763819.582973:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763819.582980:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763819.582986:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832663168, portal 10
00000100:00000001:1.0:1787763819.582990:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763819.582994:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787763819.583008:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832663168/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763844 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.583022:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763819.583029:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763819.583035:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763819.583041:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832663168, offset 0
00000400:00000200:1.0:1787763819.583049:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763819.583059:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763819.583068:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763819.583074:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763819.583080:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763819.583085:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763819.583090:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763819.583096:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58416:58416:256:4294967295] 192.168.203.110@tcp LPNI seq info [58416:58416:8:4294967295]
00000400:00000200:1.0:1787763819.583104:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763819.583118:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763819.583128:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763819.583170:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763819.583182:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763819.583188:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1787763819.583197:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763819.583203:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:1.0:1787763819.583211:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763819.583215:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763819.583222:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763819.583230:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763819.583237:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763819.583241:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763819.583247:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763819.583255:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832663168/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763844 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.583271:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763819.583278:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832663168/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763844 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.583293:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763819.583297:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763819.583305:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763819.583311:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763819.583317:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763819.583321:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316206614
00000100:00000001:1.0:1787763819.583326:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316207860
00000100:00000001:1.0:1787763819.583331:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316206614
00000100:00000001:1.0:1787763819.583335:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316207860
00000100:00000001:1.0:1787763819.583340:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763819.583344:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763819.583348:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763819.583357:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763819/real 1787763819]  req@ffff9751f8659a40 x1874599832663168/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763819.583375:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763819.583378:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763819.583382:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763819.583387:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763819.583391:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763819.583396:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763819.583403:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832663168/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763819.583422:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832663168/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.583438:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763819.583442:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763819.583446:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763819.583449:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763819.583456:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832663168/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.583469:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763819.583504:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832663168/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.583521:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763819.583526:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763819.583529:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763819.583532:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763819.583537:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763819.583541:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763819.583547:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763819.583554:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763819.583559:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763819.583568:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832663168/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763819.583585:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832663168:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763819.583594:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763819.583601:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832663296/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.583616:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763819.583620:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763819.583624:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763819.583627:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763819.583632:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763819.583639:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832663296:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763819.583646:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763819.583650:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763819.583654:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763819.583657:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763819.583665:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763819.583669:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763819.583677:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763819.583682:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787763819.583688:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787763819.583694:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832663296, portal 4
00000100:00000001:1.0:1787763819.583698:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763819.583701:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787763819.583713:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832663296/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763844 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.583727:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763819.583731:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763819.583736:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000200:1.0:1787763819.583741:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832663296, offset 0
00000400:00000200:1.0:1787763819.583750:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763819.583758:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763819.583766:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763819.583772:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763819.583778:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763819.583783:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763819.583788:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763819.583795:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58417:58417:256:4294967295] 192.168.203.110@tcp LPNI seq info [58417:58417:8:4294967295]
00000400:00000200:1.0:1787763819.583803:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763819.583818:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763819.583828:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763819.583834:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000800:00000200:1.0:1787763819.583843:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763819.583849:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:1.1:1787763819.583857:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763819.583863:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:1.0:1787763819.583871:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763819.583875:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:1.0:1787763819.583882:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763819.583889:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763819.583895:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:1.0:1787763819.583900:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:1.0:1787763819.583905:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763819.583913:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832663296/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763844 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.583930:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763819.583936:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832663296/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763844 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.583950:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763819.583953:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763819.583960:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763819.583967:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763819.583974:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763819.583978:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316206631
00000100:00000001:1.0:1787763819.583984:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316207860
00000100:00000001:1.0:1787763819.583988:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316206631
00000100:00000001:1.0:1787763819.583994:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316207860
00000100:00000001:1.0:1787763819.583999:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763819.584003:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763819.584007:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763819.584016:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763819/real 1787763819]  req@ffff9751f8658000 x1874599832663296/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763819.584035:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763819.584038:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763819.584042:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787763819.584048:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:1.0:1787763819.584052:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763819.584057:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763819.584064:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832663296/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763819.584083:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832663296/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.584099:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763819.584102:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763819.584106:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763819.584108:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763819.584117:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832663296/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.584131:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763819.584184:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832663296/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.584200:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763819.584206:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763819.584210:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763819.584213:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763819.584218:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763819.584221:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763819.584227:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763819.584234:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763819.584239:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763819.584247:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832663296/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763819.584265:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832663296:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763819.584273:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763819.584278:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763819.584284:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832663040/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.584299:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763819.584303:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763819.584309:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb800.
02000000:00000001:1.0:1787763819.584314:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763819.584318:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763819.584321:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763819.584325:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763819.584329:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:1.0:1787763819.584335:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763819.584338:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763819.584341:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:1.0:1787763819.584347:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763819.584350:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763819.584353:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763819.584361:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832663168/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.584375:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763819.584378:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763819.584382:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763819.584387:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763819.584389:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763819.584392:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763819.584397:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763819.584400:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:1.0:1787763819.584404:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763819.584406:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763819.584409:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787763819.584413:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763819.584416:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763819.584419:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763819.584427:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832663296/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763844 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763819.584440:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763819.584442:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763819.584445:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763819.584450:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763819.584452:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763819.584455:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763819.584460:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763819.584463:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:1.0:1787763819.584467:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763819.584470:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763819.584472:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:1.0:1787763819.584502:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763819.584504:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763819.584508:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763819.584516:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763819.584519:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763819.584523:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763819.584526:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763820.220239:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763820.220248:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763820.220251:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763820.220261:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763820.220261:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763820.220273:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763820.220274:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763820.220282:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763820.220284:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763820.220292:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763820.220294:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763820.220301:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763820.220303:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763820.220312:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763820.220314:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763820.220425:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763820.220430:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763820.220433:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763820.604251:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763820.604260:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763820.604263:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763820.604269:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763820.604273:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763820.604279:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763820.604284:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763820.604287:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763820.604291:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763820.860225:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316207860, at 4316207872
00000100:00000010:3.0:1787763820.860245:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4420.
00000100:00000040:2.0:1787763820.860406:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316207860, at 4316207872
00000100:00000010:2.0:1787763820.860427:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4ba0.
00010000:00000001:3.0:1787763821.244283:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763821.244291:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763821.244299:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763821.244300:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763821.244304:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763821.244314:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763821.244317:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763821.244320:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763821.244325:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763821.244336:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763821.244343:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763821.244348:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763821.244353:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763821.244356:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763821.244361:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763821.244391:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763821.244396:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763821.244400:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763821.628252:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763821.628262:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763821.628265:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763821.628270:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763821.628275:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763821.628278:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763821.628280:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763821.628283:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763821.628286:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763822.268242:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763822.268252:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763822.268255:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763822.268262:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763822.268267:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763822.268273:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763822.268275:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763822.268278:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763822.268285:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763822.268286:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763822.268288:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763822.268304:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763822.268307:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763822.268314:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763822.268338:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763822.268342:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763822.268346:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763822.268350:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763822.268353:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763822.268356:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763822.268359:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763822.268442:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763822.268448:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763822.268452:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763822.652232:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763822.652239:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763822.652242:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763822.652247:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763822.652251:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763822.652255:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763822.652257:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763822.652260:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763822.652262:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763823.292273:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763823.292284:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763823.292287:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763823.292293:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763823.292299:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763823.292300:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763823.292307:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763823.292311:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763823.292313:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763823.292320:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763823.292322:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763823.292336:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763823.292338:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763823.292343:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763823.292349:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763823.292351:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763823.292356:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763823.292360:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763823.292369:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763823.292374:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763823.292380:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763823.292410:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763823.292418:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763823.292423:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763823.676306:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763823.676318:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763823.676321:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763823.676328:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763823.676334:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763823.676339:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763823.676343:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763823.676346:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763823.676350:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763824.316210:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763824.316212:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763824.316218:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763824.316220:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763824.316225:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763824.316228:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763824.316231:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763824.316232:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763824.316239:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763824.316240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763824.316243:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763824.316245:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763824.316249:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763824.316252:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763824.316255:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763824.316292:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763824.316296:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763824.316299:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763824.700309:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763824.700319:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763824.700322:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763824.700328:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1787763824.700330:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763824.700339:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1787763824.700346:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787763824.700347:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763824.700356:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763824.700357:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787763824.700359:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763824.700367:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763824.700369:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763824.700380:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763824.700391:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21539
00000400:00000200:0.0:1787763824.700402:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763824.700412:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763824.700417:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763824.700422:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763824.700425:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763824.700428:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763824.700432:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763824.700436:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763824.700469:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763824.700472:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763824.700477:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763824.700480:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763824.700484:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763824.700487:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763824.700490:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763824.700493:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763824.700495:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763824.700498:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763824.700502:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763824.700505:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763824.700509:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763824.700522:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763824.700526:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763824.700528:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763824.700531:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763824.700546:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763824.700550:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763824.700554:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763824.700556:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763824.700560:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763824.700563:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763824.700569:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763824.700579:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763824.700586:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763824.700598:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832663424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763824.700611:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763824.700615:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763824.700621:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832663424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763824.700655:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763824.700661:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763824.700684:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763824.700686:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763824.700688:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763824.700693:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763824.700697:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21539
00000400:00000200:0.0:1787763824.700703:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763824.700708:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763824.700712:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763824.700716:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763824.700720:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787763824.700721:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1787763824.700723:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763824.700725:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787763824.700727:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1787763824.700729:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:1.0:1787763824.700730:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1787763824.700731:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763824.700734:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763824.700737:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000040:1.0:1787763824.700739:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832663424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787763824.700740:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763824.700743:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763824.700748:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763824.700752:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787763824.700752:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:0.0:1787763824.700755:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763824.700755:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763824.700758:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:1.0:1787763824.700760:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:0.0:1787763824.700761:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:1.0:1787763824.700764:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:0.0:1787763824.700765:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:1.0:1787763824.700768:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763824.700769:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763824.700773:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:1.0:1787763824.700773:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832663424:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763824.700779:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763824.700780:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763824.700783:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763824.700784:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1787763824.700786:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1787763824.700788:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1787763824.700791:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1787763824.700791:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:0.0:1787763824.700796:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763824.700798:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca800.
00000100:00000001:0.0:1787763824.700799:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763824.700801:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763824.700804:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763824.700807:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000200:1.0:1787763824.700808:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787763824.700810:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763824.700812:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763824.700813:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000100:00000001:0.0:1787763824.700815:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763824.700817:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763824.700821:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763824.700822:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
02000000:00000010:0.0:1787763824.700825:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763824.700830:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787763824.700830:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832663424, portal 25
00000100:00000001:1.0:1787763824.700834:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763824.700837:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000100:00100000:0.0:1787763824.700838:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832663552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1787763824.700846:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832663424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763824.700850:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763824.700853:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763824.700858:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763824.700860:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832663552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787763824.700863:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763824.700867:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933198.
00000100:00000001:0.0:1787763824.700873:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787763824.700873:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832663424, offset 0
00000100:00080000:0.0:1787763824.700877:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763824.700882:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:1.0:1787763824.700883:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763824.700885:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763824.700887:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763824.700890:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1787763824.700893:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763824.700901:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00080000:0.0:1787763824.700905:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21539
00000400:00000200:1.0:1787763824.700909:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763824.700914:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763824.700919:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763824.700923:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763824.700929:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58418:58418:256:4294967295] 192.168.203.110@tcp LPNI seq info [58418:58418:8:4294967295]
00000400:00000200:1.0:1787763824.700936:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763824.700940:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763824.700944:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763824.700948:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1787763824.700950:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:0.0:1787763824.700953:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763824.700956:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.0:1787763824.700959:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787763824.700960:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763824.700963:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763824.700967:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000010:1.0:1787763824.700967:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1f00.
00000020:00000040:0.0:1787763824.700969:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763824.700971:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763824.700974:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000200:1.0:1787763824.700976:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787763824.700977:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763824.700981:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:1.1:1787763824.700983:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763824.700985:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763824.700988:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763824.700990:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763824.700993:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763824.700997:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763824.701001:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763824.701004:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763824.701024:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787763824.701027:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763824.701028:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763824.701032:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763824.701033:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763824.701036:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763824.701038:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316212979
00000008:00100000:0.0:1787763824.701039:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763824.701043:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763824.701046:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4300.
00000100:00000001:0.0:1787763824.701047:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763824.701052:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763824.701057:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000001:1.0:1787763824.701057:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
02000000:00000001:0.0:1787763824.701062:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763824.701066:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763824.701069:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1787763824.701069:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316212979
02000000:00000001:0.0:1787763824.701072:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000010:1.0:1787763824.701075:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4120.
02000000:00000001:0.0:1787763824.701076:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763824.701082:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
02000000:00000010:0.0:1787763824.701083:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787763824.701090:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763824.701090:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763824.701095:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763824.701099:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832663680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763824.701099:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763824.701104:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763824.701113:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763824.701115:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763824.701118:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763824.701119:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1787763824.701124:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1787763824.701128:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832663680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763824.701128:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763824.701131:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763824.701138:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832663552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763824.701180:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763824.701187:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787763824.701189:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763824.701193:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763824.701194:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
02000000:00000001:1.0:1787763824.701197:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:0.0:1787763824.701199:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:1.0:1787763824.701201:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:0.0:1787763824.701206:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
02000000:00000001:1.0:1787763824.701206:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763824.701209:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763824.701212:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:1.0:1787763824.701213:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832663552:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763824.701217:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1787763824.701220:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:0.0:1787763824.701222:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21539
02000000:00000001:1.0:1787763824.701225:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763824.701228:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763824.701229:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:1.0:1787763824.701231:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000400:00000200:0.0:1787763824.701236:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
02000000:00000010:1.0:1787763824.701239:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca400.
00000100:00080000:0.0:1787763824.701240:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
02000000:00000001:1.0:1787763824.701243:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763824.701246:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:1.0:1787763824.701250:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763824.701251:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763824.701256:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000010:1.0:1787763824.701258:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210630.
00000100:00000001:0.0:1787763824.701261:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763824.701265:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000010:1.0:1787763824.701265:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933d48.
00000020:00000040:0.0:1787763824.701268:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763824.701271:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000200:1.0:1787763824.701271:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832663552, portal 10
00000020:00000040:0.0:1787763824.701275:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000001:1.0:1787763824.701275:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763824.701278:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
00000020:00000001:0.0:1787763824.701280:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763824.701285:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:1.0:1787763824.701291:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832663552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763824.701292:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763824.701297:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763824.701302:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763824.701305:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1787763824.701307:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:1.0:1787763824.701310:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000020:00000040:0.0:1787763824.701312:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000400:00000010:1.0:1787763824.701315:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933ee0.
00000100:00080000:0.0:1787763824.701318:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000200:1.0:1787763824.701320:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832663552, offset 0
00000100:00000001:0.0:1787763824.701323:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763824.701326:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787763824.701329:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
02000000:00000001:0.0:1787763824.701331:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763824.701335:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763824.701336:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763824.701339:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763824.701344:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000400:00000200:1.0:1787763824.701344:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000008:00000001:0.0:1787763824.701349:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763824.701349:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1787763824.701352:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763824.701355:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000010:0.0:1787763824.701358:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000400:00000200:1.0:1787763824.701360:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000020:00000040:0.0:1787763824.701362:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000400:00000200:1.0:1787763824.701365:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
02000000:00000001:0.0:1787763824.701367:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763824.701370:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763824.701371:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58419:58419:255:4294967295] 192.168.203.110@tcp LPNI seq info [58419:58419:7:4294967295]
00000100:00000001:0.0:1787763824.701374:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763824.701376:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763824.701379:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763824.701379:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000010:0.0:1787763824.701386:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763824.701392:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763824.701392:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00100000:0.0:1787763824.701401:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832663808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:1.0:1787763824.701401:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763824.701407:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1b00.
00000100:00000001:0.0:1787763824.701415:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:1.0:1787763824.701415:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763824.701419:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787763824.701422:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763824.701427:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832663808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763824.701427:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763824.701434:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000100:00000001:0.0:1787763824.701466:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763824.701467:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763824.701472:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316211744
00000100:00000040:0.0:1787763824.701491:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21549)
00000100:00000001:1.0:1787763824.701493:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316212979
00000100:00000001:1.0:1787763824.701496:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316211744
00000100:00000001:1.0:1787763824.701499:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316212979
00000100:00000001:1.0:1787763824.701503:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763824.701506:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763824.701510:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763824.701513:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763824.701518:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763824.701520:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1787763824.701524:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763824.701526:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763824.701528:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763824.701534:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832663680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763824.701564:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763824.701568:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763824.701572:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763824.701576:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763824.701580:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763824.701588:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832663680:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763824.701596:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763824.701601:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763824.701604:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763824.701608:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763824.701615:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb800.
02000000:00000001:1.0:1787763824.701635:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763824.701641:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763824.701645:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210d10.
00000400:00000010:1.0:1787763824.701653:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787763824.701658:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832663680, portal 10
00000100:00000001:1.0:1787763824.701662:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763824.701664:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00000040:1.0:1787763824.701674:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832663680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763824.701685:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763824.701689:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763824.701693:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933770.
00000100:00000200:1.0:1787763824.701697:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832663680, offset 0
00000400:00000200:1.0:1787763824.701704:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763824.701711:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763824.701716:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763824.701721:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763824.701725:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763824.701729:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763824.701733:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763824.701738:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58420:58420:254:4294967295] 192.168.203.110@tcp LPNI seq info [58420:58420:6:4294967295]
00000400:00000200:1.0:1787763824.701742:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763824.701752:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763824.701760:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763824.701765:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1300.
00000800:00000200:1.0:1787763824.701772:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763824.701777:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787763824.701782:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763824.701786:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763824.701791:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763824.701794:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316211732
00000100:00000001:1.0:1787763824.701797:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316212979
00000100:00000001:1.0:1787763824.701800:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316211732
00000100:00000001:1.0:1787763824.701804:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316212979
00000100:00000001:1.0:1787763824.701806:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763824.701810:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763824.701814:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763824.701820:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f0c0 x1874599832663808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763824.701830:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763824.701832:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763824.701835:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763824.701837:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763824.701840:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763824.701844:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832663808:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763824.701849:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763824.701851:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763824.701854:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763824.701856:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763824.701861:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c8800.
02000000:00000001:1.0:1787763824.701864:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763824.701869:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763824.701872:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787763824.701878:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79332a8.
00000100:00000200:1.0:1787763824.701883:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832663808, portal 4
00000100:00000001:1.0:1787763824.701886:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763824.701888:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049530560 : -115096660021056 : ffff9751f876f0c0)
00000100:00000040:1.0:1787763824.701897:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f0c0 x1874599832663808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763824.701951:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763824.701955:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763824.701959:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79337f8.
00000100:00000200:1.0:1787763824.701963:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832663808, offset 0
00000400:00000200:1.0:1787763824.701970:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763824.701977:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763824.701983:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763824.701988:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763824.701992:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763824.701996:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763824.702000:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763824.702004:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58421:58421:253:4294967295] 192.168.203.110@tcp LPNI seq info [58421:58421:5:4294967295]
00000400:00000200:1.0:1787763824.702011:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763824.702020:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763824.702027:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763824.702034:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1700.
00000800:00000200:1.0:1787763824.702040:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763824.702046:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787763824.702050:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763824.702054:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763824.702059:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763824.702062:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316211749
00000100:00000001:1.0:1787763824.702066:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316212979
00000100:00000001:1.0:1787763824.702069:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316211749
00000100:00000001:1.0:1787763824.702072:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316212979
00000100:00000001:1.0:1787763824.702075:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763824.702079:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763824.702083:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763824.702086:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763824.702092:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763824.702094:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1787763824.702097:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763824.702099:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763824.702101:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763824.702104:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763825.340206:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763825.340213:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763825.340215:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763825.340220:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763825.340224:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763825.340225:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763825.340227:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763825.340230:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763825.340232:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763825.340236:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763825.340237:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763825.340250:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763825.340252:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763825.340255:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763825.340258:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763825.340260:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763825.340262:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763825.340264:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763825.340266:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763825.340268:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763825.340269:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763825.340326:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763825.340330:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763825.340333:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763825.724281:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763825.724287:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763825.724289:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763825.724294:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763825.724297:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763825.724300:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763825.724302:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763825.724304:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763825.724306:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763825.980229:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316212979, at 4316212992
00000100:00000010:3.0:1787763825.980244:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4120.
00000100:00000040:2.0:1787763825.980264:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316212979, at 4316212992
00000100:00000010:2.0:1787763825.980272:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4300.
00000100:00000001:2.0:1787763826.364207:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763826.364216:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763826.364218:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763826.364218:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763826.364224:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763826.364225:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763826.364231:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763826.364233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763826.364234:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763826.364237:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763826.364238:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763826.364243:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763826.364245:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763826.364247:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763826.364249:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763826.364296:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763826.364300:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763826.364303:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763826.748254:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763826.748260:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763826.748261:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763826.748266:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763826.748268:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763826.748271:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763826.748273:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763826.748275:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763826.748277:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763827.388260:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763827.388269:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763827.388270:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763827.388277:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763827.388282:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763827.388283:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763827.388292:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763827.388293:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763827.388298:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763827.388306:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763827.388316:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763827.388322:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763827.388328:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763827.388333:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763827.388341:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763827.388446:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763827.388452:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763827.388457:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763827.772340:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763827.772347:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763827.772350:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763827.772356:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763827.772360:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763827.772364:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763827.772367:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763827.772369:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763827.772371:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:0.0:1787763827.772468:0:257563:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.110:988
00000400:00000100:0.0:1787763827.772510:0:257563:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.110@tcp at host 192.168.203.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:0.0:1787763827.772530:0:257563:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763827.772533:0:257563:0:(router.c:1791:lnet_notify()) 192.168.203.10@tcp notifying 192.168.203.110@tcp: down
00000400:00000200:0.0:1787763827.772537:0:257563:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763827.772540:0:257563:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:0.0:1787763827.772544:0:257563:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:0.0:1787763827.772548:0:257563:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1f00.
00000400:00000200:0.0:1787763827.772554:0:257563:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1787763827.772559:0:257563:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1787763827.772566:0:257563:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933198
00000400:00000010:0.0:1787763827.772569:0:257563:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933198.
00000100:00000001:0.0:1787763827.772574:0:257563:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787763827.772588:0:257563:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f876e1c0 x1874599832663424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763827.772609:0:257563:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763827.772614:0:257563:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832663424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763849 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763827.772623:0:257563:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763827.772625:0:257563:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:0.0:1787763827.772636:0:257563:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:0.0:1787763827.772639:0:257563:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1b00.
00000400:00000200:0.0:1787763827.772643:0:257563:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1787763827.772648:0:257563:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1787763827.772652:0:257563:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933ee0
00000400:00000010:0.0:1787763827.772655:0:257563:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933ee0.
00000100:00000001:1.0:1787763827.772655:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787763827.772658:0:257563:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:1.0:1787763827.772661:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000200:0.0:1787763827.772667:0:257563:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f876ed00 x1874599832663552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763827.772670:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:0.0:1787763827.772679:0:257563:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763827.772683:0:257563:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832663552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763849 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763827.772688:0:257563:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763827.772690:0:257563:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:0.0:1787763827.772697:0:257563:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000100:00100000:1.0:1787763827.772700:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763824/real 1787763827]  req@ffff9751f876e1c0 x1874599832663424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:0.0:1787763827.772701:0:257563:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1300.
00000400:00000200:0.0:1787763827.772706:0:257563:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1787763827.772716:0:257563:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1787763827.772723:0:257563:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933770
00000400:00000010:0.0:1787763827.772726:0:257563:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933770.
00000400:00000001:1.0:1787763827.772728:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:0.0:1787763827.772730:0:257563:0:(events.c:34:request_out_callback()) Process entered
00000400:00000001:1.0:1787763827.772732:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000200:0.0:1787763827.772738:0:257563:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f876e580 x1874599832663680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1787763827.772744:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000100:00000001:0.0:1787763827.772748:0:257563:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000400:00000200:1.0:1787763827.772752:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000100:00000040:0.0:1787763827.772754:0:257563:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832663680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763849 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1787763827.772758:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:1.0:1787763827.772765:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:0.0:1787763827.772769:0:257563:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763827.772772:0:257563:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000200:1.0:1787763827.772775:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832663424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:0.0:1787763827.772783:0:257563:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:0.0:1787763827.772790:0:257563:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1700.
00000100:00000200:1.0:1787763827.772796:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832663424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787763827.772798:0:257563:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1787763827.772809:0:257563:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1787763827.772816:0:257563:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79337f8
00000100:00000001:1.0:1787763827.772817:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000400:00000010:0.0:1787763827.772821:0:257563:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79337f8.
00000100:00000001:0.0:1787763827.772826:0:257563:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:1.0:1787763827.772829:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000200:0.0:1787763827.772834:0:257563:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f876f0c0 x1874599832663808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763827.772836:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763827.772841:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763827.772851:0:257563:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00100000:1.0:1787763827.772852:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832663424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763827.772859:0:257563:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f0c0 x1874599832663808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763849 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763827.772873:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763827.772877:0:257563:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763827.772881:0:257563:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:1.0:1787763827.772886:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832663424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763827.772901:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763827.772905:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763827.772907:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763827.772909:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763827.772912:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763827.772915:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763827.772919:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763827.772924:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763827.772926:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763827.772931:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832663424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763827.772939:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832663424:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763827.772945:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763827.772948:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763827.772951:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832663424/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763827.772959:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763827.772962:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763827.772966:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca800.
02000000:00000001:1.0:1787763827.772971:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763827.772974:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763827.772976:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763827.772979:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763827.772982:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014800.
02000000:00000001:1.0:1787763827.772986:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763827.772988:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763827.772990:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:1.0:1787763827.772995:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763827.772997:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763827.772999:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763827.773002:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763827.773003:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763827.773005:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763827.773009:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763824/real 1787763827]  req@ffff9751f876e580 x1874599832663680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763827.773018:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763827.773020:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763827.773022:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210d10.
00000400:00000200:1.0:1787763827.773025:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787763827.773027:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787763827.773029:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763827.773033:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832663680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763827.773041:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832663680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763827.773048:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763827.773050:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763827.773052:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763827.773053:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763827.773057:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832663680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763827.773063:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763827.773068:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832663680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763827.773075:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763827.773078:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763827.773080:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763827.773081:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763827.773083:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763827.773085:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763827.773088:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763827.773092:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763827.773094:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763827.773099:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832663680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763827.773107:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832663680:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763827.773112:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763827.773115:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763824/real 1787763827]  req@ffff9751f876f0c0 x1874599832663808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763827.773123:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763827.773124:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763827.773126:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787763827.773129:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79332a8
00000400:00000010:1.0:1787763827.773130:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79332a8.
00000100:00000001:1.0:1787763827.773156:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763827.773160:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f0c0 x1874599832663808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763827.773168:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f0c0 x1874599832663808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763827.773174:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763827.773176:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763827.773178:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763827.773180:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763827.773185:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f0c0 x1874599832663808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763827.773192:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763827.773214:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f0c0 x1874599832663808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763827.773227:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763827.773231:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763827.773233:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763827.773234:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763827.773236:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763827.773238:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763827.773240:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763827.773246:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763827.773248:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763827.773253:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f0c0 x1874599832663808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763827.773261:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832663808:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763827.773266:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763827.773270:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763824/real 1787763827]  req@ffff9751f876ed00 x1874599832663552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763827.773278:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763827.773280:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763827.773282:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210630.
00000400:00000200:1.0:1787763827.773285:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933d48
00000400:00000010:1.0:1787763827.773287:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933d48.
00000100:00000001:1.0:1787763827.773289:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763827.773292:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832663552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763827.773301:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832663552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763827.773307:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763827.773310:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763827.773312:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763827.773313:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763827.773316:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832663552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763827.773323:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763827.773327:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832663552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763827.773334:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763827.773337:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763827.773340:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763827.773342:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763827.773344:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763827.773358:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763827.773361:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763827.773365:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763827.773367:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763827.773371:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832663552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763827.773376:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832663552:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763827.773379:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763827.773381:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763827.773383:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832663680/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763827.773387:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763827.773388:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763827.773390:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb800.
02000000:00000001:1.0:1787763827.773393:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763827.773395:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763827.773397:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763827.773400:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763827.773401:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:1.0:1787763827.773404:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763827.773406:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763827.773407:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:1.0:1787763827.773411:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763827.773414:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763827.773415:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763827.773438:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f0c0 x1874599832663808/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763827.773451:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763827.773453:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763827.773455:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c8800.
02000000:00000001:1.0:1787763827.773458:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763827.773460:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763827.773462:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763827.773464:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763827.773466:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015c00.
02000000:00000001:1.0:1787763827.773470:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763827.773471:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763827.773473:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f0c0.
00000100:00000001:1.0:1787763827.773477:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763827.773478:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763827.773479:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763827.773483:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832663552/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763849 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763827.773489:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763827.773490:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763827.773492:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763827.773494:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763827.773495:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763827.773496:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763827.773498:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763827.773499:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016800.
02000000:00000001:1.0:1787763827.773502:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763827.773504:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763827.773505:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:1.0:1787763827.773508:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763827.773509:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763827.773510:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763827.773517:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763827.773519:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763827.773521:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763827.773522:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763828.412222:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763828.412223:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763828.412229:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763828.412230:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763828.412234:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763828.412236:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763828.412238:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763828.412239:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763828.412245:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763828.412247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763828.412249:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763828.412250:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763828.412254:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763828.412257:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763828.412259:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763828.412306:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763828.412310:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763828.412312:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763828.796279:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763828.796285:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763828.796287:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763828.796291:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763828.796294:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763828.796308:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763828.796310:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763828.796312:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763828.796314:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763829.436300:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763829.436303:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763829.436312:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763829.436313:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763829.436319:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763829.436323:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763829.436326:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763829.436329:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763829.436337:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763829.436340:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763829.436346:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763829.436347:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763829.436359:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763829.436364:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763829.436368:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763829.436472:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763829.436479:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763829.436483:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763829.820230:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763829.820238:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763829.820245:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763829.820248:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763829.820253:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763829.820259:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21544
00000100:00000001:3.0:1787763829.820266:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000400:00000200:0.0:1787763829.820268:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763829.820271:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763829.820274:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763829.820277:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00100000:3.0:1787763829.820279:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:0.0:1787763829.820282:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787763829.820284:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763829.820299:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763829.820300:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763829.820303:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763829.820305:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787763829.820306:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763829.820310:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763829.820311:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763829.820315:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763829.820319:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763829.820320:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763829.820322:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763829.820326:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763829.820328:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763829.820330:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763829.820333:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763829.820334:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763829.820335:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763829.820337:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763829.820339:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763829.820342:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763829.820343:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763829.820345:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763829.820348:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763829.820350:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763829.820352:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763829.820353:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763829.820358:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000020:00000040:0.0:1787763829.820361:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763829.820363:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763829.820365:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763829.820378:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763829.820380:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763829.820383:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763829.820390:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763829.820395:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763829.820402:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832663936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763829.820422:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763829.820426:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763829.820430:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832663936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763829.820458:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763829.820461:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763829.820463:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763829.820465:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763829.820469:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763829.820471:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763829.820473:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21544
00000400:00000200:0.0:1787763829.820476:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763829.820479:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763829.820481:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763829.820483:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763829.820484:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763829.820486:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763829.820487:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763829.820488:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763829.820489:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763829.820491:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763829.820492:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763829.820494:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763829.820495:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763829.820497:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763829.820498:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763829.820499:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763829.820500:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763829.820502:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763829.820504:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763829.820506:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763829.820508:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763829.820513:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763829.820514:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763829.820516:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763829.820520:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1787763829.820520:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000008:00000001:0.0:1787763829.820522:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763829.820526:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763829.820527:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763829.820531:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000010:0.0:1787763829.820534:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787763829.820538:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000100:00000040:1.0:1787763829.820541:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cf00 x1874599832663936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763829.820542:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763829.820545:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763829.820551:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763829.820554:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763829.820557:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763829.820561:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000010:0.0:1787763829.820564:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:1.0:1787763829.820565:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763829.820571:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763829.820572:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763829.820578:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00100000:0.0:1787763829.820583:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832664064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1787763829.820584:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763829.820589:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832663936:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763829.820604:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763829.820611:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1787763829.820615:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1787763829.820615:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763829.820620:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763829.820622:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:0.0:1787763829.820629:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832664064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1787763829.820630:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763829.820635:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763829.820646:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763829.820650:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763829.820652:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210630.
00000100:00080000:0.0:1787763829.820656:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:1.0:1787763829.820661:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933d48.
00000100:00080000:0.0:1787763829.820662:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763829.820666:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763829.820669:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:1.0:1787763829.820672:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832663936, portal 25
00000100:00000001:0.0:1787763829.820673:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1787763829.820678:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00080000:0.0:1787763829.820679:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21544
00000100:00000001:1.0:1787763829.820683:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049521920 : -115096660029696 : ffff9751f876cf00)
00000400:00000200:0.0:1787763829.820689:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763829.820696:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000040:1.0:1787763829.820696:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cf00 x1874599832663936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763854 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787763829.820702:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763829.820709:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763829.820714:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787763829.820716:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763829.820718:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763829.820723:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:1.0:1787763829.820726:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000020:00000001:0.0:1787763829.820727:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763829.820732:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000400:00000010:1.0:1787763829.820732:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79332a8.
00000020:00000001:0.0:1787763829.820736:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000200:1.0:1787763829.820740:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832663936, offset 0
00000020:00000040:0.0:1787763829.820741:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763829.820746:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763829.820749:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:1.0:1787763829.820752:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787763829.820757:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763829.820761:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763829.820765:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:1.0:1787763829.820766:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763829.820771:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763829.820776:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:1.0:1787763829.820778:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00080000:0.0:1787763829.820783:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763829.820787:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763829.820788:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:0.0:1787763829.820793:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787763829.820797:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
02000000:00000001:0.0:1787763829.820799:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1787763829.820804:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
02000000:00000001:0.0:1787763829.820805:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763829.820810:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:1.0:1787763829.820811:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000008:00100000:0.0:1787763829.820813:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:1.0:1787763829.820819:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58422:58422:256:4294967295] 192.168.203.110@tcp LPNI seq info [58422:58422:8:4294967295]
00000008:00000001:0.0:1787763829.820820:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763829.820825:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763829.820832:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000010:0.0:1787763829.820833:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
00000020:00000040:0.0:1787763829.820838:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763829.820842:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763829.820845:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763829.820850:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:1.0:1787763829.820851:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
02000000:00000001:0.0:1787763829.820854:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763829.820858:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787763829.820864:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
02000000:00000010:0.0:1787763829.820866:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787763829.820871:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1787763829.820871:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1a00.
00000100:00100000:0.0:1787763829.820882:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832664192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:1.0:1787763829.820884:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763829.820892:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:0.0:1787763829.820895:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763829.820900:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787763829.820901:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763829.820908:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832664192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.1:1787763829.820909:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1787763829.820922:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763829.820926:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1787763829.820928:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1a00.
00000100:00000040:0.0:1787763829.820930:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787763829.820935:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:0.0:1787763829.820937:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763829.820942:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787763829.820943:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:0.0:1787763829.820950:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763829.820952:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1787763829.820953:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79332a8
00000100:00080000:0.0:1787763829.820955:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:1.0:1787763829.820958:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79332a8.
00000100:00000001:0.0:1787763829.820959:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763829.820966:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21544
00000100:00000001:1.0:1787763829.820966:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000400:00000200:0.0:1787763829.820975:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000200:1.0:1787763829.820976:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cf00 x1874599832663936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763854 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787763829.820981:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763829.820985:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763829.820993:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763829.820997:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787763829.820997:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763829.821001:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000040:1.0:1787763829.821006:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cf00 x1874599832663936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763854 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763829.821007:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763829.821013:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763829.821015:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763829.821019:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763829.821023:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000001:1.0:1787763829.821025:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763829.821028:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:1.0:1787763829.821030:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:0.0:1787763829.821033:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763829.821039:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763829.821039:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763829.821043:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787763829.821048:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:0.0:1787763829.821049:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763829.821054:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787763829.821054:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316218099
00000020:00000040:0.0:1787763829.821061:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000010:1.0:1787763829.821064:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4a20.
00000100:00080000:0.0:1787763829.821065:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763829.821071:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763829.821073:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787763829.821074:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
02000000:00000001:0.0:1787763829.821077:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763829.821081:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763829.821084:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763829.821084:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316218099
00000008:00100000:0.0:1787763829.821088:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000010:1.0:1787763829.821091:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4cc0.
00000008:00000001:0.0:1787763829.821093:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763829.821098:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763829.821098:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763829.821105:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000010:0.0:1787763829.821107:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000100:00000001:1.0:1787763829.821112:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787763829.821113:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:1.0:1787763829.821118:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000001:0.0:1787763829.821120:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763829.821123:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763829.821126:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00100000:1.0:1787763829.821128:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763829/real 1787763829]  req@ffff9751f876cf00 x1874599832663936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763829.821129:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000001:1.0:1787763829.821163:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763829.821167:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:0.0:1787763829.821168:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763829.821170:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210630.
02000000:00000010:0.0:1787763829.821176:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000400:00000200:1.0:1787763829.821176:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933d48
00000400:00000010:1.0:1787763829.821181:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933d48.
00000100:00000001:0.0:1787763829.821183:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763829.821186:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00100000:0.0:1787763829.821191:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832664320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:1.0:1787763829.821194:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cf00 x1874599832663936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763829.821216:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763829.821220:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787763829.821221:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cf00 x1874599832663936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763829.821229:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832664320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.821236:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763829.821240:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763829.821246:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763829.821246:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763829.821250:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763829.821254:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21554)
00000100:00100000:1.0:1787763829.821259:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cf00 x1874599832663936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.821268:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763829.821275:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cf00 x1874599832663936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.821284:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763829.821287:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763829.821289:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763829.821290:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763829.821293:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763829.821295:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763829.821298:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763829.821303:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763829.821305:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763829.821310:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cf00 x1874599832663936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763829.821319:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832663936:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763829.821324:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763829.821326:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763829.821330:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cf00 x1874599832663936/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.821337:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763829.821340:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763829.821343:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763829.821347:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763829.821349:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763829.821351:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763829.821353:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763829.821355:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016800.
02000000:00000001:1.0:1787763829.821360:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763829.821362:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763829.821364:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cf00.
00000100:00000001:1.0:1787763829.821368:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763829.821369:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763829.821372:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763829.821378:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763829.821380:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763829.821382:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763829.821384:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763829.821385:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763829.821390:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e940 x1874599832664064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.821398:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763829.821400:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763829.821418:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763829.821421:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763829.821423:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763829.821427:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832664064:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763829.821432:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763829.821434:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763829.821436:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763829.821439:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763829.821443:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763829.821445:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763829.821450:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763829.821452:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210630.
00000400:00000010:1.0:1787763829.821457:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933d48.
00000100:00000200:1.0:1787763829.821460:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832664064, portal 10
00000100:00000001:1.0:1787763829.821462:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763829.821463:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049528640 : -115096660022976 : ffff9751f876e940)
00000100:00000040:1.0:1787763829.821470:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e940 x1874599832664064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763854 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.821476:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763829.821478:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763829.821481:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79332a8.
00000100:00000200:1.0:1787763829.821483:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832664064, offset 0
00000400:00000200:1.0:1787763829.821487:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763829.821492:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763829.821497:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763829.821500:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763829.821503:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763829.821506:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763829.821508:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763829.821511:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58423:58423:256:4294967295] 192.168.203.110@tcp LPNI seq info [58423:58423:8:4294967295]
00000400:00000200:1.0:1787763829.821514:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763829.821521:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763829.821526:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763829.821529:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1a00.
00000800:00000200:1.0:1787763829.821534:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763829.821536:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787763829.821540:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763829.821543:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1787763829.821547:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763829.821548:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1a00.
00000400:00000200:1.0:1787763829.821552:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763829.821556:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763829.821559:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79332a8
00000400:00000010:1.0:1787763829.821561:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79332a8.
00000100:00000001:1.0:1787763829.821563:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763829.821566:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e940 x1874599832664064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763854 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.821573:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763829.821576:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e940 x1874599832664064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763854 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.821583:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763829.821584:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763829.821587:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763829.821591:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763829.821594:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763829.821596:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316216864
00000100:00000001:1.0:1787763829.821599:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316218099
00000100:00000001:1.0:1787763829.821602:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316216864
00000100:00000001:1.0:1787763829.821604:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316218099
00000100:00000001:1.0:1787763829.821606:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763829.821608:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763829.821610:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763829.821614:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763829/real 1787763829]  req@ffff9751f876e940 x1874599832664064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763829.821622:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763829.821623:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763829.821625:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210630.
00000400:00000200:1.0:1787763829.821627:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933d48
00000400:00000010:1.0:1787763829.821629:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933d48.
00000100:00000001:1.0:1787763829.821632:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763829.821635:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e940 x1874599832664064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763829.821644:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e940 x1874599832664064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.821650:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763829.821652:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763829.821654:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763829.821656:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763829.821660:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e940 x1874599832664064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.821667:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763829.821672:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e940 x1874599832664064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.821680:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763829.821682:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763829.821684:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763829.821686:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763829.821688:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763829.821699:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763829.821701:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763829.821704:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763829.821705:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763829.821708:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e940 x1874599832664064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763829.821714:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832664064:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763829.821716:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763829.821718:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f840 x1874599832664192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.821723:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763829.821724:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763829.821725:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763829.821727:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763829.821728:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763829.821730:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832664192:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763829.821743:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763829.821745:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763829.821746:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763829.821747:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763829.821750:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c8800.
02000000:00000001:1.0:1787763829.821751:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763829.821754:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763829.821755:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210630.
00000400:00000010:1.0:1787763829.821758:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933d48.
00000100:00000200:1.0:1787763829.821760:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832664192, portal 10
00000100:00000001:1.0:1787763829.821761:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763829.821763:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049532480 : -115096660019136 : ffff9751f876f840)
00000100:00000040:1.0:1787763829.821767:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f840 x1874599832664192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763854 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.821771:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763829.821772:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763829.821774:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79332a8.
00000100:00000200:1.0:1787763829.821776:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832664192, offset 0
00000400:00000200:1.0:1787763829.821778:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763829.821781:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763829.821783:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763829.821786:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763829.821788:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763829.821790:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763829.821791:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763829.821793:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58424:58424:256:4294967295] 192.168.203.110@tcp LPNI seq info [58424:58424:8:4294967295]
00000400:00000200:1.0:1787763829.821796:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763829.821800:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763829.821803:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763829.821805:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1a00.
00000800:00000200:1.0:1787763829.821807:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763829.821809:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787763829.821812:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763829.821814:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1787763829.821817:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763829.821818:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1a00.
00000400:00000200:1.0:1787763829.821820:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763829.821822:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763829.821824:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79332a8
00000400:00000010:1.0:1787763829.821826:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79332a8.
00000100:00000001:1.0:1787763829.821827:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763829.821830:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f840 x1874599832664192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763854 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.821835:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763829.821837:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f840 x1874599832664192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763854 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.821842:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763829.821843:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763829.821845:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763829.821847:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763829.821849:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763829.821850:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316216852
00000100:00000001:1.0:1787763829.821852:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316218099
00000100:00000001:1.0:1787763829.821854:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316216852
00000100:00000001:1.0:1787763829.821855:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316218099
00000100:00000001:1.0:1787763829.821856:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763829.821858:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763829.821859:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763829.821861:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763829/real 1787763829]  req@ffff9751f876f840 x1874599832664192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763829.821866:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763829.821867:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763829.821868:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210630.
00000400:00000200:1.0:1787763829.821869:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933d48
00000400:00000010:1.0:1787763829.821870:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933d48.
00000100:00000001:1.0:1787763829.821871:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763829.821873:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f840 x1874599832664192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763829.821878:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f840 x1874599832664192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.821883:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763829.821884:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763829.821885:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763829.821886:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763829.821899:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f840 x1874599832664192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.821903:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763829.821906:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f840 x1874599832664192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.821910:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763829.821911:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763829.821912:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763829.821913:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763829.821914:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763829.821915:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763829.821916:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763829.821918:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763829.821920:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763829.821924:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f840 x1874599832664192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763829.821932:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832664192:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763829.821936:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763829.821939:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832664320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.821945:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763829.821946:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763829.821949:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763829.821950:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763829.821953:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763829.821956:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832664320:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763829.821959:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763829.821961:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763829.821963:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763829.821965:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763829.821968:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb800.
02000000:00000001:1.0:1787763829.821971:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763829.821974:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763829.821977:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210630.
00000400:00000010:1.0:1787763829.821980:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933d48.
00000100:00000200:1.0:1787763829.821983:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832664320, portal 4
00000100:00000001:1.0:1787763829.821985:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763829.821986:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
00000100:00000040:1.0:1787763829.821992:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832664320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763854 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.821996:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763829.821998:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763829.821999:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79332a8.
00000100:00000200:1.0:1787763829.822001:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832664320, offset 0
00000400:00000200:1.0:1787763829.822004:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763829.822006:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763829.822009:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763829.822011:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763829.822013:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763829.822015:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763829.822016:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763829.822018:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58425:58425:256:4294967295] 192.168.203.110@tcp LPNI seq info [58425:58425:8:4294967295]
00000400:00000200:1.0:1787763829.822021:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763829.822025:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763829.822028:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763829.822030:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1a00.
00000800:00000200:1.0:1787763829.822033:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763829.822035:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787763829.822037:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763829.822039:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1787763829.822042:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763829.822043:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1a00.
00000400:00000200:1.0:1787763829.822045:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763829.822048:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763829.822049:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79332a8
00000400:00000010:1.0:1787763829.822051:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79332a8.
00000100:00000001:1.0:1787763829.822052:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763829.822054:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876ed00 x1874599832664320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763854 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.822059:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763829.822061:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832664320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763854 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.822066:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763829.822067:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763829.822068:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763829.822070:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763829.822072:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763829.822073:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316216869
00000100:00000001:1.0:1787763829.822075:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316218099
00000100:00000001:1.0:1787763829.822076:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316216869
00000100:00000001:1.0:1787763829.822078:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316218099
00000100:00000001:1.0:1787763829.822079:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763829.822080:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763829.822082:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763829.822084:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763829/real 1787763829]  req@ffff9751f876ed00 x1874599832664320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763829.822088:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763829.822089:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763829.822090:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210630.
00000400:00000200:1.0:1787763829.822092:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933d48
00000400:00000010:1.0:1787763829.822093:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933d48.
00000100:00000001:1.0:1787763829.822094:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763829.822096:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832664320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763829.822102:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832664320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.822105:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763829.822106:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763829.822107:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763829.822108:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763829.822110:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832664320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.822114:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763829.822116:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832664320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.822121:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763829.822122:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763829.822123:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763829.822124:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763829.822126:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763829.822127:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763829.822128:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763829.822130:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763829.822131:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763829.822151:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832664320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763829.822156:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832664320:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763829.822158:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763829.822160:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763829.822162:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e940 x1874599832664064/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.822166:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763829.822168:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763829.822169:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763829.822171:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763829.822172:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763829.822173:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763829.822174:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763829.822176:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015c00.
02000000:00000001:1.0:1787763829.822178:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763829.822179:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763829.822180:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e940.
00000100:00000001:1.0:1787763829.822182:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763829.822183:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763829.822185:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763829.822187:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f840 x1874599832664192/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.822191:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763829.822192:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763829.822193:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c8800.
02000000:00000001:1.0:1787763829.822194:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763829.822195:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763829.822195:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763829.822197:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763829.822197:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:1.0:1787763829.822198:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763829.822199:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763829.822200:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f840.
00000100:00000001:1.0:1787763829.822201:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763829.822202:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763829.822203:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763829.822205:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832664320/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763854 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763829.822208:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763829.822209:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763829.822210:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb800.
02000000:00000001:1.0:1787763829.822212:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763829.822213:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763829.822214:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763829.822215:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763829.822216:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014800.
02000000:00000001:1.0:1787763829.822217:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763829.822217:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763829.822218:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:1.0:1787763829.822219:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763829.822220:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763829.822221:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763829.822224:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763829.822225:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763829.822226:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763829.822227:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763830.460231:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763830.460243:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763830.460249:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763830.460258:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763830.460266:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763830.460272:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763830.460274:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763830.460279:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763830.460281:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763830.460288:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763830.460291:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763830.460297:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763830.460300:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763830.460303:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763830.460309:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763830.460377:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763830.460381:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763830.460383:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763830.844374:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763830.844382:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763830.844385:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763830.844391:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763830.844395:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763830.844419:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763830.844422:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763830.844425:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763830.844428:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763831.100225:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316218099, at 4316218112
00000100:00000010:3.0:1787763831.100240:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4cc0.
00000100:00000040:2.0:1787763831.100266:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316218099, at 4316218112
00000100:00000010:2.0:1787763831.100277:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4a20.
00000100:00000001:2.0:1787763831.484213:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763831.484213:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763831.484220:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763831.484221:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763831.484225:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763831.484227:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763831.484229:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763831.484230:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763831.484235:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763831.484238:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763831.484239:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763831.484243:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763831.484244:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763831.484245:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763831.484248:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787763831.484249:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763831.484253:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763831.484259:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763831.484262:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763831.484264:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763831.484267:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763831.484340:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763831.484346:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763831.484350:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763831.868213:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763831.868219:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763831.868220:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763831.868224:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763831.868229:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763831.868231:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763831.868233:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763831.868235:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763831.868237:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763832.508207:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763832.508207:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763832.508213:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763832.508215:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763832.508218:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763832.508219:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763832.508221:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763832.508222:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763832.508228:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763832.508229:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763832.508232:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763832.508232:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763832.508236:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763832.508238:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763832.508240:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763832.508276:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763832.508280:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763832.508283:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763832.892219:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763832.892228:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763832.892231:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763832.892237:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763832.892242:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763832.892245:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763832.892248:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763832.892250:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763832.892253:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763833.532234:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763833.532237:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763833.532240:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763833.532243:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763833.532244:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763833.532246:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00100000:2.0:1787763833.532250:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763833.532251:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763833.532257:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763833.532257:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787763833.532261:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763833.532261:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763833.532263:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763833.532266:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763833.532266:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763833.532270:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763833.532272:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763833.532274:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763833.532274:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763833.532282:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763833.532283:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763833.532343:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763833.532347:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763833.532349:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763833.916202:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763833.916209:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763833.916211:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763833.916216:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763833.916219:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763833.916221:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763833.916223:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763833.916226:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763833.916228:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763834.556218:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763834.556225:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763834.556232:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763834.556235:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763834.556236:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763834.556241:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763834.556241:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763834.556246:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763834.556246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763834.556252:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763834.556256:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763834.556259:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763834.556261:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763834.556263:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763834.556265:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763834.556336:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763834.556340:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763834.556343:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763834.940235:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763834.940243:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763834.940246:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763834.940248:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:0.0:1787763834.940253:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:2.0:1787763834.940256:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763834.940260:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763834.940261:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1787763834.940264:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1787763834.940266:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763834.940267:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763834.940271:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763834.940274:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1787763834.940275:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763834.940283:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21549
00000400:00000200:2.0:1787763834.940293:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763834.940300:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763834.940303:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763834.940307:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763834.940310:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763834.940312:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763834.940315:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763834.940319:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763834.940322:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787763834.940325:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763834.940329:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787763834.940331:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787763834.940333:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787763834.940335:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763834.940337:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763834.940338:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763834.940339:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763834.940342:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787763834.940344:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763834.940345:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763834.940347:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763834.940350:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763834.940352:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763834.940354:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787763834.940356:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763834.940363:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000020:00000040:2.0:1787763834.940366:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787763834.940381:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763834.940383:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763834.940385:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763834.940387:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763834.940390:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763834.940397:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bbc00.
00000100:00000001:2.0:1787763834.940402:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763834.940410:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832664448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763834.940418:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763834.940421:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763834.940424:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832664448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763834.940440:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763834.940442:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763834.940445:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787763834.940446:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763834.940447:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763834.940449:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763834.940451:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21549
00000400:00000200:2.0:1787763834.940453:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763834.940456:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763834.940458:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763834.940461:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763834.940463:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763834.940465:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763834.940466:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787763834.940467:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1787763834.940468:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763834.940471:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00000001:1.0:1787763834.940473:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1787763834.940473:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763834.940475:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000001:1.0:1787763834.940476:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1787763834.940478:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:2.0:1787763834.940480:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:2.0:1787763834.940485:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000040:1.0:1787763834.940486:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832664448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763834.940487:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763834.940491:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763834.940495:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763834.940499:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:2.0:1787763834.940504:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763834.940505:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763834.940507:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763834.940508:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763834.940510:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763834.940512:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763834.940515:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763834.940516:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763834.940519:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763834.940520:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763834.940522:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:1.0:1787763834.940526:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832664448:192.168.203.110@tcp:250:kworker.0
00000008:00100000:2.0:1787763834.940527:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787763834.940533:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763834.940534:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1787763834.940535:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763834.940539:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763834.940541:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763834.940541:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
00000020:00000040:2.0:1787763834.940546:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763834.940547:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1787763834.940551:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763834.940552:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763834.940553:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000010:1.0:1787763834.940554:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb800.
02000000:00000001:2.0:1787763834.940555:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763834.940558:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763834.940558:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763834.940563:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5b9800.
00000400:00000200:1.0:1787763834.940566:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763834.940569:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787763834.940571:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210630.
00000100:00100000:2.0:1787763834.940575:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832664576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1787763834.940580:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933d48.
00000100:00000001:2.0:1787763834.940582:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763834.940585:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787763834.940588:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832664448, portal 25
00000100:00000040:2.0:1787763834.940589:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832664576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.940593:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763834.940596:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000001:2.0:1787763834.940603:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763834.940605:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832664448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763859 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787763834.940606:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763834.940612:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1787763834.940615:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763834.940617:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763834.940621:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1787763834.940622:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763834.940628:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21549
00000100:00000040:1.0:1787763834.940629:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763834.940633:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79332a8.
00000400:00000200:2.0:1787763834.940634:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000200:1.0:1787763834.940637:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832664448, offset 0
00000400:00000200:2.0:1787763834.940638:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763834.940643:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763834.940645:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763834.940648:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787763834.940649:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000040:2.0:1787763834.940652:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763834.940655:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763834.940656:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763834.940657:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787763834.940658:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:1.0:1787763834.940661:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:2.0:1787763834.940661:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:2.0:1787763834.940666:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:2.0:1787763834.940668:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:1.0:1787763834.940673:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000040:2.0:1787763834.940674:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763834.940678:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763834.940681:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:1.0:1787763834.940682:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1787763834.940683:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763834.940687:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:1.0:1787763834.940689:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00080000:2.0:1787763834.940692:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763834.940695:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763834.940696:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
02000000:00000001:2.0:1787763834.940698:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787763834.940701:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
02000000:00000001:2.0:1787763834.940702:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1787763834.940706:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58426:58426:256:4294967295] 192.168.203.110@tcp LPNI seq info [58426:58426:8:4294967295]
02000000:00000001:2.0:1787763834.940707:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763834.940711:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787763834.940715:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:1.0:1787763834.940718:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000008:00000001:2.0:1787763834.940719:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763834.940722:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763834.940726:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000400:00000200:1.0:1787763834.940729:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000020:00000040:2.0:1787763834.940730:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787763834.940735:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763834.940737:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763834.940740:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:1.0:1787763834.940743:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
02000000:00000001:2.0:1787763834.940743:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763834.940745:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763834.940750:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bb800.
00000100:00000001:2.0:1787763834.940752:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1787763834.940753:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1a00.
00000100:00100000:2.0:1787763834.940757:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832664704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763834.940764:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:1.0:1787763834.940766:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787763834.940767:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763834.940773:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832664704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.0:1787763834.940774:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1787763834.940783:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763834.940786:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:1.1:1787763834.940788:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:2.0:1787763834.940789:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:2.0:1787763834.940795:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:1.1:1787763834.940796:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000100:00080000:2.0:1787763834.940803:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787763834.940806:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000100:1.0:1787763834.940807:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00080000:2.0:1787763834.940808:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000010:1.0:1787763834.940810:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1a00.
00000100:00000001:2.0:1787763834.940812:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763834.940816:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21549
00000400:00000200:1.0:1787763834.940818:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763834.940819:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763834.940825:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763834.940832:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763834.940834:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763834.940835:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:1.0:1787763834.940837:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79332a8
00000100:00000001:2.0:1787763834.940837:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:1.0:1787763834.940842:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79332a8.
00000100:00000040:2.0:1787763834.940842:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763834.940846:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787763834.940849:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000020:00000001:2.0:1787763834.940849:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763834.940853:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787763834.940856:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000200:1.0:1787763834.940857:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832664448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763859 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787763834.940858:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787763834.940863:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:2.0:1787763834.940867:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:2.0:1787763834.940873:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763834.940875:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787763834.940876:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763834.940877:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763834.940880:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:1.0:1787763834.940882:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832664448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763859 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787763834.940882:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:2.0:1787763834.940888:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763834.940891:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763834.940892:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763834.940893:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763834.940895:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763834.940896:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787763834.940899:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1787763834.940902:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1787763834.940903:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763834.940908:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763834.940909:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000010:2.0:1787763834.940911:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000020:00000040:2.0:1787763834.940914:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:2.0:1787763834.940916:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763834.940917:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763834.940919:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763834.940919:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763834.940921:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763834.940925:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763834.940927:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5ba000.
00000400:00000200:1.0:1787763834.940929:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763834.940930:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763834.940934:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316223219
00000100:00100000:2.0:1787763834.940936:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832664832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:1.0:1787763834.940944:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c41e0.
00000100:00000001:2.0:1787763834.940944:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763834.940946:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763834.940951:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832664832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.940955:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787763834.940958:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763834.940963:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21559)
00000100:00000001:1.0:1787763834.940964:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316223219
00000100:00000010:1.0:1787763834.940968:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4480.
00000100:00000001:1.0:1787763834.940971:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787763834.940974:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763834.940976:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763834.940978:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763834.940983:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763834/real 1787763834]  req@ffff9751f86583c0 x1874599832664448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763834.940990:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763834.940992:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763834.940994:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210630.
00000400:00000200:1.0:1787763834.940997:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933d48
00000400:00000010:1.0:1787763834.940999:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933d48.
00000100:00000001:1.0:1787763834.941001:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763834.941005:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832664448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763834.941015:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832664448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941023:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763834.941025:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763834.941028:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763834.941030:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763834.941035:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832664448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941042:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763834.941045:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832664448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941050:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763834.941052:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763834.941053:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763834.941054:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763834.941056:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763834.941058:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763834.941060:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763834.941062:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763834.941064:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763834.941067:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832664448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763834.941072:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832664448:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763834.941076:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763834.941077:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763834.941080:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832664448/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941084:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763834.941086:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763834.941088:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb800.
02000000:00000001:1.0:1787763834.941090:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763834.941091:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763834.941093:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787763834.941094:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763834.941096:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:1.0:1787763834.941099:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763834.941100:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763834.941102:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:1.0:1787763834.941104:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763834.941105:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763834.941106:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763834.941111:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763834.941112:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763834.941113:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763834.941114:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763834.941116:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763834.941118:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832664576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941123:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763834.941124:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763834.941125:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763834.941127:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763834.941128:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763834.941130:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832664576:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763834.941150:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763834.941152:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763834.941153:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763834.941155:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763834.941157:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb800.
02000000:00000001:1.0:1787763834.941158:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763834.941161:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763834.941163:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210630.
00000400:00000010:1.0:1787763834.941166:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933d48.
00000100:00000200:1.0:1787763834.941168:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832664576, portal 10
00000100:00000001:1.0:1787763834.941170:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763834.941171:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787763834.941176:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832664576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763859 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941181:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763834.941182:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763834.941184:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79332a8.
00000100:00000200:1.0:1787763834.941186:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832664576, offset 0
00000400:00000200:1.0:1787763834.941188:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763834.941191:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763834.941194:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763834.941196:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763834.941198:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763834.941200:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763834.941201:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763834.941203:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58427:58427:256:4294967295] 192.168.203.110@tcp LPNI seq info [58427:58427:8:4294967295]
00000400:00000200:1.0:1787763834.941206:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763834.941211:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763834.941214:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763834.941217:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1a00.
00000800:00000200:1.0:1787763834.941220:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763834.941222:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787763834.941225:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763834.941227:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1787763834.941229:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763834.941231:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1a00.
00000400:00000200:1.0:1787763834.941233:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763834.941236:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763834.941238:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79332a8
00000400:00000010:1.0:1787763834.941239:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79332a8.
00000100:00000001:1.0:1787763834.941241:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763834.941243:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832664576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763859 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941248:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763834.941250:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832664576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763859 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941259:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763834.941260:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763834.941262:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763834.941265:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787763834.941267:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763834.941268:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316221984
00000100:00000001:1.0:1787763834.941270:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316223219
00000100:00000001:1.0:1787763834.941272:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316221984
00000100:00000001:1.0:1787763834.941273:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316223219
00000100:00000001:1.0:1787763834.941275:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763834.941276:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763834.941277:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763834.941279:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763834/real 1787763834]  req@ffff9751f8658000 x1874599832664576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763834.941284:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763834.941285:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763834.941286:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210630.
00000400:00000200:1.0:1787763834.941287:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933d48
00000400:00000010:1.0:1787763834.941288:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933d48.
00000100:00000001:1.0:1787763834.941290:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763834.941292:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832664576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763834.941297:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832664576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941301:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763834.941302:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763834.941303:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763834.941303:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763834.941306:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832664576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941311:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763834.941313:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832664576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941318:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763834.941319:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763834.941320:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763834.941321:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763834.941322:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763834.941332:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763834.941333:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763834.941335:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763834.941337:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763834.941339:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832664576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763834.941344:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832664576:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763834.941346:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763834.941348:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832664704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941353:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763834.941354:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763834.941355:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763834.941356:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763834.941357:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763834.941359:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832664704:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763834.941361:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763834.941362:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763834.941364:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763834.941365:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763834.941382:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c8800.
02000000:00000001:1.0:1787763834.941385:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763834.941389:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763834.941391:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210630.
00000400:00000010:1.0:1787763834.941393:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933d48.
00000100:00000200:1.0:1787763834.941395:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832664704, portal 10
00000100:00000001:1.0:1787763834.941396:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763834.941397:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787763834.941402:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832664704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763859 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941407:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763834.941410:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763834.941412:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79332a8.
00000100:00000200:1.0:1787763834.941415:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832664704, offset 0
00000400:00000200:1.0:1787763834.941419:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763834.941423:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763834.941427:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763834.941429:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763834.941432:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763834.941434:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763834.941437:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763834.941440:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58428:58428:256:4294967295] 192.168.203.110@tcp LPNI seq info [58428:58428:8:4294967295]
00000400:00000200:1.0:1787763834.941445:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763834.941452:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763834.941457:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763834.941460:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1a00.
00000800:00000200:1.0:1787763834.941463:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763834.941465:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787763834.941468:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763834.941470:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1787763834.941473:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763834.941474:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1a00.
00000400:00000200:1.0:1787763834.941477:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763834.941479:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763834.941481:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79332a8
00000400:00000010:1.0:1787763834.941482:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79332a8.
00000100:00000001:1.0:1787763834.941484:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763834.941487:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832664704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763859 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941493:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763834.941494:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832664704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763859 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941499:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763834.941500:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763834.941502:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763834.941504:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787763834.941507:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763834.941508:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316221972
00000100:00000001:1.0:1787763834.941510:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316223219
00000100:00000001:1.0:1787763834.941511:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316221972
00000100:00000001:1.0:1787763834.941513:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316223219
00000100:00000001:1.0:1787763834.941514:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763834.941515:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763834.941517:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763834.941519:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763834/real 1787763834]  req@ffff9751f8659a40 x1874599832664704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763834.941523:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763834.941524:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763834.941526:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210630.
00000400:00000200:1.0:1787763834.941527:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933d48
00000400:00000010:1.0:1787763834.941528:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933d48.
00000100:00000001:1.0:1787763834.941530:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763834.941532:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832664704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763834.941537:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832664704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941541:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763834.941542:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763834.941543:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763834.941544:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763834.941546:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832664704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941550:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763834.941552:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832664704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941556:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763834.941557:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763834.941558:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763834.941559:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763834.941560:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763834.941561:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763834.941563:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763834.941565:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763834.941566:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763834.941569:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832664704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763834.941574:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832664704:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787763834.941576:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787763834.941579:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832664832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941583:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763834.941584:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763834.941586:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763834.941587:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763834.941588:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763834.941590:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832664832:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763834.941592:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763834.941593:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763834.941594:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763834.941596:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787763834.941598:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763834.941599:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763834.941601:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787763834.941603:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210630.
00000400:00000010:1.0:1787763834.941605:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933d48.
00000100:00000200:1.0:1787763834.941607:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832664832, portal 4
00000100:00000001:1.0:1787763834.941608:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787763834.941609:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:1.0:1787763834.941613:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832664832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763859 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941617:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787763834.941619:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787763834.941620:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79332a8.
00000100:00000200:1.0:1787763834.941622:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832664832, offset 0
00000400:00000200:1.0:1787763834.941624:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763834.941626:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763834.941629:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763834.941631:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787763834.941632:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787763834.941634:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787763834.941636:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763834.941638:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58429:58429:256:4294967295] 192.168.203.110@tcp LPNI seq info [58429:58429:8:4294967295]
00000400:00000200:1.0:1787763834.941641:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763834.941645:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787763834.941648:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763834.941650:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1a00.
00000800:00000200:1.0:1787763834.941652:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787763834.941654:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787763834.941656:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787763834.941658:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1787763834.941661:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787763834.941662:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1a00.
00000400:00000200:1.0:1787763834.941664:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787763834.941666:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787763834.941667:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79332a8
00000400:00000010:1.0:1787763834.941669:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79332a8.
00000100:00000001:1.0:1787763834.941670:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787763834.941672:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832664832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763859 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941677:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763834.941679:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832664832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763859 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941684:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763834.941685:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763834.941686:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763834.941688:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787763834.941690:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763834.941691:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316221989
00000100:00000001:1.0:1787763834.941692:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316223219
00000100:00000001:1.0:1787763834.941694:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316221989
00000100:00000001:1.0:1787763834.941695:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316223219
00000100:00000001:1.0:1787763834.941697:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763834.941699:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763834.941700:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787763834.941702:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763834/real 1787763834]  req@ffff9751f865ad00 x1874599832664832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787763834.941707:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787763834.941708:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787763834.941708:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210630.
00000400:00000200:1.0:1787763834.941710:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933d48
00000400:00000010:1.0:1787763834.941711:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933d48.
00000100:00000001:1.0:1787763834.941712:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787763834.941713:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832664832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787763834.941719:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832664832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941723:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787763834.941724:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763834.941725:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763834.941726:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763834.941728:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832664832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941732:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763834.941734:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832664832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941741:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787763834.941743:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763834.941744:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787763834.941746:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763834.941747:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787763834.941749:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787763834.941751:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763834.941755:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787763834.941757:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787763834.941761:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832664832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763834.941770:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832664832:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787763834.941774:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763834.941776:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763834.941780:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832664576/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941787:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763834.941788:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763834.941790:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb800.
02000000:00000001:1.0:1787763834.941792:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763834.941793:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763834.941794:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763834.941795:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763834.941796:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:1.0:1787763834.941798:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763834.941799:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763834.941800:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:1.0:1787763834.941802:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763834.941803:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763834.941804:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763834.941806:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832664704/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941810:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763834.941811:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763834.941812:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c8800.
02000000:00000001:1.0:1787763834.941814:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763834.941814:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763834.941815:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787763834.941816:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763834.941818:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:1.0:1787763834.941819:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763834.941820:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763834.941821:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787763834.941822:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763834.941823:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763834.941824:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787763834.941826:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832664832/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763859 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763834.941829:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787763834.941830:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787763834.941831:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca400.
02000000:00000001:1.0:1787763834.941832:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787763834.941833:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787763834.941834:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787763834.941835:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787763834.941835:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:1.0:1787763834.941837:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787763834.941837:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787763834.941838:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:1.0:1787763834.941840:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763834.941840:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763834.941841:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763834.941844:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763834.941845:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763834.941846:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763834.941847:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763835.580235:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763835.580240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763835.580242:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763835.580245:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763835.580247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763835.580252:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763835.580255:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763835.580257:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763835.580261:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763835.580262:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763835.580264:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763835.580266:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787763835.580267:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763835.580270:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763835.580271:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763835.580273:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763835.580276:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763835.580278:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763835.580279:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763835.580283:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763835.580285:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763835.580379:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763835.580385:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763835.580387:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763835.964223:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763835.964231:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763835.964233:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763835.964239:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763835.964243:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763835.964246:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763835.964249:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763835.964252:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763835.964255:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763836.220248:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316223219, at 4316223232
00000100:00000010:3.0:1787763836.220262:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4480.
00000100:00000040:2.0:1787763836.220324:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316223219, at 4316223232
00000100:00000010:2.0:1787763836.220342:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c41e0.
00000100:00000001:2.0:1787763836.604239:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763836.604241:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763836.604245:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763836.604246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763836.604250:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763836.604252:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763836.604254:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763836.604255:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763836.604262:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763836.604262:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763836.604266:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763836.604267:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763836.604271:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763836.604273:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763836.604275:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763836.604337:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763836.604342:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763836.604346:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787763836.988257:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763836.988264:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763836.988266:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763836.988271:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763836.988274:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763836.988278:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763836.988280:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763836.988282:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763836.988284:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763837.628293:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763837.628300:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763837.628301:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763837.628308:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763837.628310:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763837.628325:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763837.628326:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763837.628331:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763837.628334:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763837.628337:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763837.628338:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763837.628340:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763837.628343:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763837.628346:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763837.628370:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763837.628418:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763837.628422:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763837.628426:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763838.012257:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763838.012264:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763838.012266:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763838.012271:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763838.012277:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763838.012280:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763838.012283:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763838.012285:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763838.012287:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763838.652255:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763838.652259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763838.652263:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763838.652267:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763838.652268:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763838.652280:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763838.652281:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763838.652287:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763838.652289:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763838.652293:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763838.652294:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763838.652296:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763838.652300:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763838.652303:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763838.652305:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763838.652374:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763838.652378:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763838.652382:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763839.036209:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763839.036216:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763839.036218:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763839.036222:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763839.036226:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763839.036229:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763839.036231:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763839.036233:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763839.036235:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763839.676237:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763839.676245:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763839.676249:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763839.676252:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763839.676255:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763839.676261:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763839.676261:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763839.676264:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763839.676269:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763839.676277:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763839.676281:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763839.676284:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763839.676286:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763839.676288:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763839.676292:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763839.676313:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763839.676318:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763839.676321:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763840.060274:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763840.060285:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763840.060287:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763840.060291:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1787763840.060295:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:1.0:1787763840.060307:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787763840.060308:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763840.060310:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787763840.060313:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:1.0:1787763840.060314:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763840.060317:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763840.060320:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763840.060321:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763840.060330:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763840.060349:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21554
00000400:00000200:1.0:1787763840.060359:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763840.060367:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763840.060371:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763840.060377:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763840.060381:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763840.060384:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763840.060386:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763840.060390:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763840.060392:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787763840.060395:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763840.060398:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787763840.060400:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787763840.060402:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787763840.060404:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763840.060406:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763840.060407:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763840.060409:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763840.060412:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787763840.060414:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763840.060416:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763840.060419:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763840.060422:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763840.060424:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763840.060426:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763840.060429:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763840.060435:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932940.
00000020:00000040:1.0:1787763840.060438:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787763840.060442:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763840.060444:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763840.060447:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763840.060450:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763840.060455:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763840.060462:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca400.
00000100:00000001:1.0:1787763840.060469:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763840.060480:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932940 x1874599832664960/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763840.060490:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763840.060494:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763840.060499:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000af90eaa] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932940 x1874599832664960/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763840.060521:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763840.060525:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763840.060529:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787763840.060531:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763840.060533:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763840.060536:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787763840.060539:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:1.0:1787763840.060540:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21554
00000100:00000001:3.0:1787763840.060545:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000400:00000200:1.0:1787763840.060546:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763840.060551:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000400:00000200:1.0:1787763840.060553:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763840.060556:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763840.060561:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000040:3.0:1787763840.060562:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932940 x1874599832664960/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763840.060563:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763840.060569:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763840.060573:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763840.060579:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763840.060581:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787763840.060583:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763840.060585:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787763840.060588:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787763840.060590:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787763840.060593:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763840.060595:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763840.060597:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763840.060600:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763840.060602:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787763840.060604:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763840.060606:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763840.060607:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763840.060614:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763840.060617:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763840.060618:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763840.060623:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763840.060627:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763840.060629:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763840.060632:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930780.
00000020:00000040:1.0:1787763840.060634:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763840.060637:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1787763840.060639:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:1.0:1787763840.060640:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763840.060642:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763840.060644:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763840.060648:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763840.060648:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763840.060651:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763840.060654:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000010:1.0:1787763840.060657:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8800.
02000000:00000001:3.0:1787763840.060658:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763840.060663:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763840.060668:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832664960:192.168.203.110@tcp:250:kworker.0
00000100:00100000:1.0:1787763840.060672:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930780 x1874599832665088/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763840.060680:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763840.060682:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763840.060684:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3977950] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930780 x1874599832665088/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763840.060690:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763840.060692:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763840.060695:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787763840.060696:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763840.060698:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763840.060701:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763840.060706:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21554
00000400:00000200:1.0:1787763840.060709:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763840.060713:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763840.060715:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763840.060718:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763840.060720:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763840.060725:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763840.060727:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763840.060729:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763840.060730:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787763840.060731:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763840.060732:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787763840.060734:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787763840.060735:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787763840.060737:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763840.060738:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763840.060739:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763840.060740:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763840.060741:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787763840.060743:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763840.060744:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763840.060745:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763840.060748:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763840.060749:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763840.060750:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763840.060751:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763840.060753:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763840.060754:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763840.060756:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932580.
00000020:00000040:1.0:1787763840.060759:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763840.060761:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763840.060762:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763840.060763:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763840.060764:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763840.060765:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763840.060768:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb800.
00000100:00000001:1.0:1787763840.060769:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763840.060772:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932580 x1874599832665216/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763840.060777:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763840.060779:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763840.060781:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d026924] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932580 x1874599832665216/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763840.060786:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763840.060788:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787763840.060791:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787763840.060793:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763840.060795:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787763840.060796:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763840.060796:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763840.060799:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763840.060803:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21554
00000100:00000001:3.0:1787763840.060805:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000400:00000200:1.0:1787763840.060808:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763840.060811:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
02000000:00000001:3.0:1787763840.060812:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:1.0:1787763840.060813:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763840.060818:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:3.0:1787763840.060818:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763840.060823:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763840.060826:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763840.060828:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763840.060830:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763840.060831:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787763840.060832:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763840.060833:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787763840.060835:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787763840.060837:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:1.0:1787763840.060846:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763840.060847:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763840.060850:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763840.060851:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763840.060853:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:1.0:1787763840.060855:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763840.060858:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763840.060861:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763840.060862:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:1.0:1787763840.060864:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763840.060867:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763840.060868:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613800.
00000100:00000001:1.0:1787763840.060869:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763840.060874:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
02000000:00000001:3.0:1787763840.060875:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:1.0:1787763840.060881:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763840.060883:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763840.060887:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000010:1.0:1787763840.060889:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933840.
00000400:00000010:3.0:1787763840.060892:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000020:00000040:1.0:1787763840.060893:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787763840.060898:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763840.060901:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763840.060902:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763840.060904:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:3.0:1787763840.060904:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
02000000:00000001:1.0:1787763840.060907:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763840.060912:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca800.
00000100:00000200:3.0:1787763840.060914:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832664960, portal 25
00000100:00000001:1.0:1787763840.060915:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763840.060920:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00100000:1.0:1787763840.060923:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933840 x1874599832665344/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787763840.060924:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497731904 : -115097211819712 : ffff9751d7932940)
00000100:00000040:3.0:1787763840.060936:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932940 x1874599832664960/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763865 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763840.060952:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787763840.060954:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:1.0:1787763840.060955:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763840.060960:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000040:1.0:1787763840.060962:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000083dd6d32] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933840 x1874599832665344/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1787763840.060965:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763840.060970:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832664960, offset 0
00000100:00000001:1.0:1787763840.060974:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763840.060979:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21564)
00000400:00000200:3.0:1787763840.060980:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763840.060990:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763840.060997:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763840.061003:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763840.061007:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763840.061011:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763840.061013:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763840.061016:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58430:58430:256:4294967295] 192.168.203.110@tcp LPNI seq info [58430:58430:8:4294967295]
00000400:00000200:3.0:1787763840.061020:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763840.061028:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763840.061033:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763840.061039:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763840.061047:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763840.061051:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787763840.061055:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763840.061058:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1787763840.061062:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763840.061064:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763840.061068:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763840.061073:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763840.061077:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763840.061080:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763840.061083:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763840.061087:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932940 x1874599832664960/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763865 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061095:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763840.061098:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932940 x1874599832664960/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763865 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061104:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763840.061105:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763840.061111:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763840.061114:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763840.061116:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316228339
00000100:00000010:3.0:1787763840.061120:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40e40.
00000100:00000001:3.0:1787763840.061127:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787763840.061163:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316228340
00000100:00000010:3.0:1787763840.061167:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40c00.
00000100:00000001:3.0:1787763840.061172:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787763840.061187:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763840.061190:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763840.061192:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763840.061196:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763840/real 1787763840]  req@ffff9751d7932940 x1874599832664960/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763840.061204:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763840.061206:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763840.061207:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763840.061210:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763840.061211:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763840.061213:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763840.061215:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932940 x1874599832664960/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763840.061222:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932940 x1874599832664960/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061226:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763840.061227:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763840.061229:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763840.061231:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763840.061234:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932940 x1874599832664960/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061239:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763840.061242:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932940 x1874599832664960/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061248:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763840.061250:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763840.061251:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763840.061252:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763840.061254:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763840.061255:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763840.061257:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763840.061260:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763840.061261:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763840.061264:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932940 x1874599832664960/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763840.061269:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832664960:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787763840.061274:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763840.061276:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763840.061278:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932940 x1874599832664960/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061283:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763840.061286:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763840.061288:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613800.
02000000:00000001:3.0:1787763840.061289:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763840.061291:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763840.061292:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787763840.061293:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763840.061296:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca400.
02000000:00000001:3.0:1787763840.061299:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763840.061300:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763840.061301:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932940.
00000100:00000001:3.0:1787763840.061305:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763840.061306:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763840.061307:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763840.061314:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763840.061316:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763840.061318:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763840.061319:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763840.061320:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763840.061322:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930780 x1874599832665088/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061327:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763840.061329:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763840.061330:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763840.061343:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763840.061345:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763840.061347:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832665088:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763840.061350:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763840.061351:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763840.061353:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763840.061355:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763840.061357:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613800.
02000000:00000001:3.0:1787763840.061358:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763840.061361:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763840.061363:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763840.061366:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763840.061368:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832665088, portal 10
00000100:00000001:3.0:1787763840.061370:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763840.061371:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497723264 : -115097211828352 : ffff9751d7930780)
00000100:00000040:3.0:1787763840.061376:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930780 x1874599832665088/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763865 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061381:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763840.061383:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763840.061384:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763840.061386:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832665088, offset 0
00000400:00000200:3.0:1787763840.061388:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763840.061391:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763840.061393:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763840.061396:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763840.061397:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763840.061399:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763840.061401:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763840.061403:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58431:58431:256:4294967295] 192.168.203.110@tcp LPNI seq info [58431:58431:8:4294967295]
00000400:00000200:3.0:1787763840.061406:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763840.061411:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763840.061414:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763840.061416:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763840.061419:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763840.061421:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787763840.061424:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763840.061426:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1787763840.061429:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763840.061431:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763840.061434:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763840.061436:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763840.061438:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763840.061440:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763840.061441:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763840.061444:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930780 x1874599832665088/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763865 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061449:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763840.061451:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930780 x1874599832665088/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763865 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061456:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763840.061457:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763840.061459:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763840.061462:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787763840.061464:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763840.061466:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316227104
00000100:00000001:3.0:1787763840.061468:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316228339
00000100:00000001:3.0:1787763840.061469:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316227104
00000100:00000001:3.0:1787763840.061471:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316228340
00000100:00000001:3.0:1787763840.061473:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763840.061474:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763840.061475:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763840.061478:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763840/real 1787763840]  req@ffff9751d7930780 x1874599832665088/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763840.061482:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763840.061483:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763840.061484:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763840.061486:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763840.061487:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763840.061489:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763840.061491:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930780 x1874599832665088/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763840.061497:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930780 x1874599832665088/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061501:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763840.061502:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763840.061503:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763840.061504:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763840.061506:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930780 x1874599832665088/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061510:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763840.061512:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930780 x1874599832665088/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061516:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763840.061518:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763840.061519:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763840.061520:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763840.061521:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763840.061531:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763840.061533:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763840.061535:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763840.061537:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763840.061539:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930780 x1874599832665088/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763840.061545:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832665088:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763840.061547:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763840.061549:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932580 x1874599832665216/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061558:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763840.061559:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763840.061561:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763840.061562:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763840.061563:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763840.061565:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832665216:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763840.061567:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763840.061568:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763840.061569:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763840.061570:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763840.061572:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611800.
02000000:00000001:3.0:1787763840.061573:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763840.061575:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763840.061576:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763840.061578:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763840.061579:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832665216, portal 10
00000100:00000001:3.0:1787763840.061581:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763840.061582:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497730944 : -115097211820672 : ffff9751d7932580)
00000100:00000040:3.0:1787763840.061586:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932580 x1874599832665216/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763865 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061591:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763840.061592:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763840.061594:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763840.061595:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832665216, offset 0
00000400:00000200:3.0:1787763840.061598:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763840.061600:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763840.061602:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763840.061604:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763840.061605:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763840.061607:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763840.061608:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763840.061611:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58432:58432:256:4294967295] 192.168.203.110@tcp LPNI seq info [58432:58432:8:4294967295]
00000400:00000200:3.0:1787763840.061614:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763840.061618:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763840.061621:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763840.061623:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763840.061625:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763840.061627:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787763840.061630:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763840.061631:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1787763840.061634:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763840.061636:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763840.061638:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763840.061640:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763840.061642:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763840.061643:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763840.061645:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763840.061647:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932580 x1874599832665216/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763865 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061652:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763840.061654:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932580 x1874599832665216/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763865 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061659:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763840.061659:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763840.061661:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763840.061663:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787763840.061665:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763840.061666:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316227092
00000100:00000001:3.0:1787763840.061667:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316228339
00000100:00000001:3.0:1787763840.061668:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316227092
00000100:00000001:3.0:1787763840.061670:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316228340
00000100:00000001:3.0:1787763840.061672:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763840.061673:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763840.061675:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763840.061677:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763840/real 1787763840]  req@ffff9751d7932580 x1874599832665216/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763840.061683:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763840.061683:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763840.061684:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763840.061686:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763840.061687:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763840.061688:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763840.061690:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932580 x1874599832665216/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763840.061696:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932580 x1874599832665216/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061700:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763840.061701:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763840.061702:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763840.061702:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763840.061705:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932580 x1874599832665216/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061708:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763840.061711:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932580 x1874599832665216/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061715:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763840.061716:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763840.061717:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763840.061718:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763840.061719:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763840.061720:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763840.061721:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763840.061724:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763840.061725:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763840.061728:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932580 x1874599832665216/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763840.061733:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832665216:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763840.061735:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763840.061737:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933840 x1874599832665344/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061741:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763840.061742:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763840.061743:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763840.061744:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763840.061745:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763840.061747:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832665344:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763840.061749:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763840.061750:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763840.061751:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763840.061752:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763840.061754:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610400.
02000000:00000001:3.0:1787763840.061755:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763840.061757:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763840.061758:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763840.061760:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763840.061762:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832665344, portal 4
00000100:00000001:3.0:1787763840.061764:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763840.061765:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497735744 : -115097211815872 : ffff9751d7933840)
00000100:00000040:3.0:1787763840.061768:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933840 x1874599832665344/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763865 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061773:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763840.061775:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763840.061777:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763840.061778:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832665344, offset 0
00000400:00000200:3.0:1787763840.061781:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763840.061783:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763840.061785:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763840.061787:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763840.061789:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763840.061791:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763840.061792:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763840.061794:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58433:58433:256:4294967295] 192.168.203.110@tcp LPNI seq info [58433:58433:8:4294967295]
00000400:00000200:3.0:1787763840.061797:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763840.061801:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763840.061804:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763840.061806:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763840.061808:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763840.061810:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787763840.061813:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763840.061815:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1787763840.061817:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763840.061819:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763840.061821:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763840.061823:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763840.061824:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763840.061825:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763840.061827:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763840.061829:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933840 x1874599832665344/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763865 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061834:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763840.061836:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933840 x1874599832665344/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763865 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061841:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763840.061842:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763840.061844:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763840.061846:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787763840.061848:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763840.061850:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316227109
00000100:00000001:3.0:1787763840.061852:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316228339
00000100:00000001:3.0:1787763840.061854:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316227109
00000100:00000001:3.0:1787763840.061857:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316228340
00000100:00000001:3.0:1787763840.061859:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763840.061862:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763840.061863:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763840.061867:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763840/real 1787763840]  req@ffff9751d7933840 x1874599832665344/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763840.061876:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763840.061877:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763840.061879:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763840.061882:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763840.061884:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763840.061887:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763840.061890:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933840 x1874599832665344/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763840.061900:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933840 x1874599832665344/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061907:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763840.061909:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763840.061911:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763840.061912:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763840.061915:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933840 x1874599832665344/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061920:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763840.061923:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933840 x1874599832665344/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061928:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763840.061930:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763840.061931:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763840.061932:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763840.061933:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763840.061934:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763840.061936:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763840.061938:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763840.061939:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763840.061941:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933840 x1874599832665344/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763840.061946:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832665344:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763840.061948:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763840.061950:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763840.061952:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930780 x1874599832665088/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061957:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763840.061958:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763840.061959:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613800.
02000000:00000001:3.0:1787763840.061961:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763840.061962:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763840.061963:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763840.061965:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763840.061967:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8800.
02000000:00000001:3.0:1787763840.061968:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763840.061970:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763840.061971:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930780.
00000100:00000001:3.0:1787763840.061972:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763840.061973:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763840.061975:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763840.061977:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932580 x1874599832665216/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061981:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763840.061982:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763840.061983:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611800.
02000000:00000001:3.0:1787763840.061984:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763840.061985:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763840.061986:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763840.061987:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763840.061988:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb800.
02000000:00000001:3.0:1787763840.061989:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763840.061990:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763840.061991:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932580.
00000100:00000001:3.0:1787763840.061992:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763840.061993:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763840.061994:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763840.061995:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933840 x1874599832665344/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763865 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763840.061999:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763840.062000:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763840.062000:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610400.
02000000:00000001:3.0:1787763840.062002:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763840.062002:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763840.062003:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787763840.062004:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763840.062005:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca800.
02000000:00000001:3.0:1787763840.062006:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763840.062007:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763840.062007:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933840.
00000100:00000001:3.0:1787763840.062010:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763840.062011:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763840.062012:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763840.062014:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763840.062015:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763840.062016:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763840.062018:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763840.700262:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763840.700267:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763840.700272:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763840.700272:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763840.700276:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763840.700279:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763840.700280:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763840.700282:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763840.700287:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763840.700289:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763840.700291:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763840.700292:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763840.700298:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763840.700302:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763840.700305:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763840.700358:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763840.700361:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763840.700365:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763841.084260:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763841.084269:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763841.084271:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763841.084277:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763841.084281:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763841.084285:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763841.084287:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763841.084290:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763841.084293:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763841.340267:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316228340, at 4316228352
00000100:00000010:3.0:1787763841.340282:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40c00.
00000100:00000040:2.0:1787763841.340338:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316228339, at 4316228352
00000100:00000010:2.0:1787763841.340352:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40e40.
00000100:00000001:2.0:1787763841.724197:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763841.724204:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763841.724206:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763841.724211:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763841.724215:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763841.724217:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763841.724218:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763841.724222:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763841.724223:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763841.724227:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763841.724229:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763841.724231:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763841.724232:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763841.724237:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763841.724240:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763841.724242:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763841.724244:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763841.724247:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763841.724249:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763841.724250:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763841.724252:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763841.724297:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763841.724302:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763841.724305:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763842.108212:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763842.108221:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763842.108223:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763842.108227:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763842.108231:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763842.108233:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763842.108235:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763842.108237:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763842.108239:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763842.748240:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763842.748250:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763842.748250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763842.748253:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763842.748255:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763842.748261:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763842.748263:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763842.748266:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763842.748268:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763842.748270:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763842.748273:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763842.748274:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763842.748278:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763842.748281:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763842.748284:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763842.748340:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763842.748344:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763842.748347:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763843.132229:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763843.132236:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763843.132237:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763843.132241:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763843.132244:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763843.132247:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763843.132249:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763843.132252:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763843.132254:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763843.772228:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763843.772229:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763843.772241:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763843.772242:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763843.772244:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763843.772246:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763843.772252:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763843.772253:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763843.772259:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787763843.772261:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763843.772262:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763843.772268:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763843.772270:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763843.772271:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763843.772275:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763843.772278:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763843.772279:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763843.772286:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763843.772286:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763843.772291:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763843.772298:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763843.772359:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763843.772364:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763843.772368:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763844.156251:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763844.156258:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763844.156261:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763844.156265:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763844.156269:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763844.156272:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763844.156274:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763844.156276:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763844.156278:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763844.796211:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763844.796218:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763844.796221:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763844.796227:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763844.796228:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763844.796232:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763844.796233:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763844.796239:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763844.796240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763844.796243:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763844.796246:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763844.796249:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763844.796250:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763844.796253:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763844.796256:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763844.796295:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763844.796309:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763844.796312:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763845.180229:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:1.0:1787763845.180232:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763845.180239:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787763845.180240:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763845.180242:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787763845.180243:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763845.180246:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1787763845.180248:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787763845.180252:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763845.180253:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763845.180257:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:1.0:1787763845.180260:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21559
00000100:00000001:2.0:1787763845.180261:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763845.180265:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763845.180268:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763845.180269:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763845.180278:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763845.180280:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763845.180283:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763845.180286:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763845.180289:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763845.180291:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763845.180293:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763845.180295:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787763845.180311:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763845.180315:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787763845.180317:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787763845.180319:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787763845.180321:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763845.180323:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763845.180324:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763845.180325:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763845.180328:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787763845.180331:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763845.180333:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763845.180335:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763845.180337:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763845.180339:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763845.180341:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763845.180343:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763845.180350:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933840.
00000020:00000040:1.0:1787763845.180353:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787763845.180356:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763845.180357:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763845.180359:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763845.180361:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763845.180364:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763845.180373:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8000.
00000100:00000001:1.0:1787763845.180378:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763845.180385:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933840 x1874599832665472/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763845.180393:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763845.180396:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763845.180399:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000083dd6d32] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933840 x1874599832665472/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763845.180417:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763845.180419:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763845.180421:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787763845.180423:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763845.180424:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763845.180426:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763845.180428:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21559
00000400:00000200:1.0:1787763845.180431:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763845.180435:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763845.180437:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763845.180439:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763845.180441:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787763845.180441:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:1.0:1787763845.180445:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763845.180448:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787763845.180449:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:1.0:1787763845.180450:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763845.180452:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00000001:3.0:1787763845.180453:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1787763845.180454:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763845.180457:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787763845.180459:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787763845.180461:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787763845.180465:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763845.180467:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763845.180467:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933840 x1874599832665472/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787763845.180469:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763845.180473:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763845.180477:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787763845.180479:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763845.180480:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763845.180481:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763845.180486:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763845.180490:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763845.180493:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787763845.180498:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00100000:1.0:1787763845.180499:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787763845.180501:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:1.0:1787763845.180502:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763845.180505:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763845.180507:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000010:1.0:1787763845.180508:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932580.
02000000:00000001:3.0:1787763845.180511:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:1.0:1787763845.180512:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763845.180518:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763845.180518:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763845.180520:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763845.180523:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763845.180525:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00100000:3.0:1787763845.180526:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832665472:192.168.203.110@tcp:250:kworker.0
02000000:00000001:1.0:1787763845.180527:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763845.180532:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9000.
00000100:00000001:3.0:1787763845.180534:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1787763845.180536:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763845.180541:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00100000:1.0:1787763845.180542:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932580 x1874599832665600/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1787763845.180546:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763845.180552:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:3.0:1787763845.180553:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1787763845.180554:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763845.180558:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d026924] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932580 x1874599832665600/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1787763845.180562:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610400.
00000100:00000001:1.0:1787763845.180565:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763845.180567:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763845.180568:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763845.180572:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787763845.180573:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763845.180575:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1787763845.180578:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763845.180579:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763845.180584:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21559
00000400:00000010:3.0:1787763845.180585:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000200:1.0:1787763845.180590:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763845.180593:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763845.180595:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000010:3.0:1787763845.180595:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000040:1.0:1787763845.180602:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763845.180603:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763845.180605:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000200:3.0:1787763845.180606:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832665472, portal 25
00000100:00000001:1.0:1787763845.180607:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763845.180611:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1787763845.180611:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763845.180614:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497735744 : -115097211815872 : ffff9751d7933840)
00000020:00000040:1.0:1787763845.180615:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787763845.180618:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763845.180620:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787763845.180622:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787763845.180625:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:3.0:1787763845.180625:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933840 x1874599832665472/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763870 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787763845.180633:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763845.180636:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763845.180640:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763845.180641:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:1.0:1787763845.180642:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763845.180646:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00000040:3.0:1787763845.180647:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00080000:1.0:1787763845.180650:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000010:3.0:1787763845.180652:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000001:1.0:1787763845.180653:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763845.180657:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:3.0:1787763845.180657:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832665472, offset 0
02000000:00000001:1.0:1787763845.180659:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763845.180662:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763845.180663:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763845.180666:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:3.0:1787763845.180667:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000008:00000001:1.0:1787763845.180671:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763845.180673:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763845.180676:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930780.
00000400:00000200:3.0:1787763845.180677:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:1.0:1787763845.180679:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763845.180682:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763845.180684:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763845.180686:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:1.0:1787763845.180687:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763845.180690:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763845.180693:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763845.180693:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763845.180700:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
02000000:00000010:1.0:1787763845.180701:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb400.
00000400:00000200:3.0:1787763845.180705:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1787763845.180708:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763845.180708:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763845.180714:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58434:58434:256:4294967295] 192.168.203.110@tcp LPNI seq info [58434:58434:8:4294967295]
00000100:00100000:1.0:1787763845.180719:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930780 x1874599832665728/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787763845.180720:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763845.180734:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:1.0:1787763845.180735:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763845.180740:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787763845.180741:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000040:1.0:1787763845.180748:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3977950] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930780 x1874599832665728/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:3.0:1787763845.180748:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000100:00000001:1.0:1787763845.180756:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787763845.180759:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:1.0:1787763845.180760:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787763845.180766:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:3.0:1787763845.180769:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000100:00080000:1.0:1787763845.180770:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763845.180775:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787763845.180777:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763845.180779:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763845.180783:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:3.1:1787763845.180784:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00080000:1.0:1787763845.180787:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21559
00000800:00000200:3.1:1787763845.180791:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000400:00000200:1.0:1787763845.180792:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763845.180798:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763845.180800:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000100:3.0:1787763845.180804:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000040:1.0:1787763845.180805:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763845.180808:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000010:3.0:1787763845.180808:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000100:00000040:1.0:1787763845.180813:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763845.180815:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:3.0:1787763845.180816:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:1.0:1787763845.180817:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763845.180819:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787763845.180822:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763845.180825:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000400:00000200:3.0:1787763845.180825:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:1.0:1787763845.180827:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787763845.180830:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000200:3.0:1787763845.180832:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000100:00000040:1.0:1787763845.180834:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000010:3.0:1787763845.180836:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:1.0:1787763845.180837:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763845.180841:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763845.180842:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:1.0:1787763845.180843:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763845.180846:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000200:3.0:1787763845.180849:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933840 x1874599832665472/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763870 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787763845.180851:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763845.180856:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763845.180860:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763845.180864:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763845.180867:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763845.180867:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787763845.180870:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:3.0:1787763845.180872:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933840 x1874599832665472/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763870 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:1.0:1787763845.180874:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1787763845.180879:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763845.180882:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763845.180888:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932940.
00000100:00000001:3.0:1787763845.180889:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763845.180893:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000040:1.0:1787763845.180898:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:3.0:1787763845.180899:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763845.180902:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763845.180905:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763845.180906:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763845.180908:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1787763845.180909:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:1.0:1787763845.180910:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763845.180917:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca000.
00000100:00000001:3.0:1787763845.180917:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316233459
00000100:00000001:1.0:1787763845.180921:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763845.180926:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40c00.
00000100:00100000:1.0:1787763845.180927:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932940 x1874599832665856/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763845.180935:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763845.180937:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763845.180937:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000040:1.0:1787763845.180943:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000af90eaa] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932940 x1874599832665856/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.180946:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316233459
00000100:00000010:3.0:1787763845.180952:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d408a0.
00000100:00000001:1.0:1787763845.180953:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763845.180957:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000040:1.0:1787763845.180959:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21569)
00000100:00000001:3.0:1787763845.180962:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763845.180965:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763845.180967:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763845.180972:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763845/real 1787763845]  req@ffff9751d7933840 x1874599832665472/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763845.180979:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763845.180981:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763845.180982:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763845.180984:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763845.180985:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763845.180987:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763845.180990:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933840 x1874599832665472/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763845.180996:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933840 x1874599832665472/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181000:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763845.181002:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763845.181003:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763845.181005:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763845.181007:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933840 x1874599832665472/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181012:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763845.181016:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933840 x1874599832665472/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181025:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763845.181027:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763845.181028:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763845.181029:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763845.181031:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763845.181032:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763845.181034:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763845.181036:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763845.181038:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763845.181040:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933840 x1874599832665472/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763845.181046:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832665472:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787763845.181049:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763845.181050:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763845.181052:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933840 x1874599832665472/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181057:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763845.181059:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763845.181061:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610400.
02000000:00000001:3.0:1787763845.181062:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763845.181064:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763845.181065:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787763845.181067:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763845.181068:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8000.
02000000:00000001:3.0:1787763845.181071:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763845.181072:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763845.181073:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933840.
00000100:00000001:3.0:1787763845.181076:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763845.181077:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763845.181078:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763845.181084:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763845.181085:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763845.181086:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763845.181087:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763845.181088:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763845.181091:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932580 x1874599832665600/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181095:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763845.181096:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763845.181098:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763845.181099:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763845.181101:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763845.181102:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832665600:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763845.181104:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763845.181106:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763845.181107:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763845.181108:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763845.181110:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610400.
02000000:00000001:3.0:1787763845.181111:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763845.181115:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763845.181117:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763845.181121:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763845.181124:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832665600, portal 10
00000100:00000001:3.0:1787763845.181126:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763845.181128:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497730944 : -115097211820672 : ffff9751d7932580)
00000100:00000040:3.0:1787763845.181157:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932580 x1874599832665600/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763870 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181164:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763845.181167:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763845.181170:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763845.181172:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832665600, offset 0
00000400:00000200:3.0:1787763845.181175:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763845.181178:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763845.181182:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763845.181184:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763845.181186:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763845.181188:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763845.181190:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763845.181192:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58435:58435:256:4294967295] 192.168.203.110@tcp LPNI seq info [58435:58435:8:4294967295]
00000400:00000200:3.0:1787763845.181195:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763845.181199:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763845.181202:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763845.181205:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763845.181208:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763845.181210:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787763845.181213:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763845.181215:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1787763845.181217:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763845.181219:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763845.181221:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763845.181223:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763845.181225:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763845.181227:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763845.181228:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763845.181231:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932580 x1874599832665600/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763870 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181237:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763845.181239:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932580 x1874599832665600/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763870 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181243:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763845.181244:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763845.181247:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763845.181249:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787763845.181252:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763845.181253:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316232224
00000100:00000001:3.0:1787763845.181256:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316233459
00000100:00000001:3.0:1787763845.181257:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316232224
00000100:00000001:3.0:1787763845.181259:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316233459
00000100:00000001:3.0:1787763845.181261:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763845.181262:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763845.181264:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763845.181266:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763845/real 1787763845]  req@ffff9751d7932580 x1874599832665600/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763845.181271:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763845.181271:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763845.181273:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763845.181274:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763845.181275:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763845.181276:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763845.181278:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932580 x1874599832665600/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763845.181284:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932580 x1874599832665600/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181287:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763845.181288:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763845.181289:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763845.181290:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763845.181293:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932580 x1874599832665600/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181311:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763845.181316:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932580 x1874599832665600/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181320:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763845.181322:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763845.181324:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763845.181325:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763845.181326:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763845.181336:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763845.181338:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763845.181340:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763845.181342:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763845.181344:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932580 x1874599832665600/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763845.181349:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832665600:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763845.181351:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763845.181353:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930780 x1874599832665728/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181357:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763845.181359:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763845.181360:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763845.181361:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763845.181363:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763845.181364:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832665728:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763845.181366:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763845.181367:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763845.181369:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763845.181370:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763845.181372:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611800.
02000000:00000001:3.0:1787763845.181374:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763845.181376:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763845.181378:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763845.181380:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763845.181382:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832665728, portal 10
00000100:00000001:3.0:1787763845.181383:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763845.181385:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497723264 : -115097211828352 : ffff9751d7930780)
00000100:00000040:3.0:1787763845.181389:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930780 x1874599832665728/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763870 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181393:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763845.181395:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763845.181396:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763845.181398:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832665728, offset 0
00000400:00000200:3.0:1787763845.181400:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763845.181403:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763845.181405:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763845.181407:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763845.181408:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763845.181410:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763845.181411:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763845.181413:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58436:58436:256:4294967295] 192.168.203.110@tcp LPNI seq info [58436:58436:8:4294967295]
00000400:00000200:3.0:1787763845.181416:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763845.181420:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763845.181423:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763845.181425:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763845.181428:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763845.181430:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787763845.181432:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763845.181434:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1787763845.181437:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763845.181438:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763845.181441:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763845.181444:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763845.181447:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763845.181449:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763845.181452:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763845.181455:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930780 x1874599832665728/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763870 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181463:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763845.181467:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930780 x1874599832665728/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763870 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181474:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763845.181476:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763845.181480:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763845.181483:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787763845.181486:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763845.181488:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316232212
00000100:00000001:3.0:1787763845.181501:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316233459
00000100:00000001:3.0:1787763845.181506:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316232212
00000100:00000001:3.0:1787763845.181508:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316233459
00000100:00000001:3.0:1787763845.181509:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763845.181511:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763845.181512:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763845.181516:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763845/real 1787763845]  req@ffff9751d7930780 x1874599832665728/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763845.181521:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763845.181522:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763845.181523:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763845.181525:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763845.181527:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763845.181529:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763845.181532:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930780 x1874599832665728/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763845.181541:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930780 x1874599832665728/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181548:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763845.181549:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763845.181551:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763845.181553:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763845.181557:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930780 x1874599832665728/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181565:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763845.181569:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930780 x1874599832665728/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181576:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763845.181578:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763845.181579:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763845.181579:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763845.181581:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763845.181582:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763845.181584:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763845.181586:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763845.181587:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763845.181589:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930780 x1874599832665728/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763845.181595:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832665728:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763845.181598:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763845.181600:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932940 x1874599832665856/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181604:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763845.181605:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763845.181606:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763845.181608:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763845.181609:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763845.181611:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832665856:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763845.181613:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763845.181615:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763845.181616:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763845.181618:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763845.181620:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613800.
02000000:00000001:3.0:1787763845.181622:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763845.181624:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763845.181626:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763845.181628:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763845.181631:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832665856, portal 4
00000100:00000001:3.0:1787763845.181632:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763845.181633:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497731904 : -115097211819712 : ffff9751d7932940)
00000100:00000040:3.0:1787763845.181637:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932940 x1874599832665856/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763870 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181642:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763845.181643:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763845.181644:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763845.181646:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832665856, offset 0
00000400:00000200:3.0:1787763845.181648:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763845.181651:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763845.181654:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763845.181656:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763845.181657:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763845.181659:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763845.181661:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763845.181663:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58437:58437:256:4294967295] 192.168.203.110@tcp LPNI seq info [58437:58437:8:4294967295]
00000400:00000200:3.0:1787763845.181666:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763845.181670:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763845.181672:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763845.181674:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763845.181677:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763845.181679:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787763845.181682:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763845.181683:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1787763845.181686:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763845.181688:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763845.181690:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763845.181692:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763845.181694:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763845.181695:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763845.181697:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763845.181699:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932940 x1874599832665856/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763870 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181705:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763845.181707:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932940 x1874599832665856/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763870 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181711:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763845.181712:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763845.181714:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763845.181716:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787763845.181718:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763845.181719:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316232229
00000100:00000001:3.0:1787763845.181721:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316233459
00000100:00000001:3.0:1787763845.181722:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316232229
00000100:00000001:3.0:1787763845.181724:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316233459
00000100:00000001:3.0:1787763845.181725:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763845.181726:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763845.181727:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763845.181729:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763845/real 1787763845]  req@ffff9751d7932940 x1874599832665856/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763845.181734:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763845.181735:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763845.181735:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763845.181737:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763845.181738:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763845.181740:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763845.181742:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932940 x1874599832665856/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763845.181747:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932940 x1874599832665856/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181751:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763845.181752:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763845.181754:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763845.181755:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763845.181759:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932940 x1874599832665856/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181766:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763845.181770:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932940 x1874599832665856/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181778:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763845.181781:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763845.181782:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763845.181784:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763845.181785:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763845.181787:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763845.181790:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763845.181794:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763845.181796:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763845.181801:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932940 x1874599832665856/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763845.181807:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832665856:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763845.181810:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763845.181812:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763845.181814:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932580 x1874599832665600/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181819:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763845.181820:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763845.181821:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610400.
02000000:00000001:3.0:1787763845.181823:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763845.181824:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763845.181825:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763845.181826:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763845.181828:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9000.
02000000:00000001:3.0:1787763845.181829:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763845.181830:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763845.181831:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932580.
00000100:00000001:3.0:1787763845.181834:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763845.181835:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763845.181836:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763845.181838:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930780 x1874599832665728/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181842:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763845.181843:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763845.181844:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611800.
02000000:00000001:3.0:1787763845.181845:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763845.181846:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763845.181847:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763845.181848:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763845.181849:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb400.
02000000:00000001:3.0:1787763845.181850:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763845.181850:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763845.181851:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930780.
00000100:00000001:3.0:1787763845.181853:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763845.181853:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763845.181854:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763845.181856:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932940 x1874599832665856/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763870 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763845.181860:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763845.181862:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763845.181863:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613800.
02000000:00000001:3.0:1787763845.181865:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763845.181865:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763845.181866:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787763845.181867:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763845.181868:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca000.
02000000:00000001:3.0:1787763845.181869:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763845.181870:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763845.181871:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932940.
00000100:00000001:3.0:1787763845.181872:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763845.181873:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763845.181874:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763845.181876:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763845.181877:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763845.181879:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763845.181880:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763845.820202:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763845.820211:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763845.820215:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763845.820219:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763845.820221:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763845.820224:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763845.820226:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763845.820229:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763845.820230:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763845.820233:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763845.820236:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763845.820238:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763845.820240:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763845.820241:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787763845.820244:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763845.820245:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763845.820250:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763845.820250:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763845.820252:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763845.820257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763845.820258:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763845.820313:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763845.820316:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763845.820319:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763846.204220:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763846.204227:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763846.204229:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763846.204234:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763846.204239:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763846.204242:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763846.204245:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763846.204248:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763846.204251:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763846.460182:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316233459, at 4316233472
00000100:00000010:3.0:1787763846.460197:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d408a0.
00000100:00000040:2.0:1787763846.460234:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316233459, at 4316233472
00000100:00000010:2.0:1787763846.460244:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40c00.
00000100:00000001:2.0:1787763846.844219:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763846.844219:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763846.844225:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763846.844227:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763846.844230:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763846.844235:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763846.844235:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763846.844239:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763846.844240:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763846.844244:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763846.844246:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763846.844247:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763846.844250:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763846.844251:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763846.844254:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763846.844310:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763846.844314:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763846.844317:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763847.228209:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763847.228217:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763847.228220:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763847.228225:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763847.228230:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763847.228234:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763847.228237:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763847.228240:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763847.228243:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763847.868209:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763847.868213:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763847.868220:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763847.868225:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763847.868225:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763847.868233:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763847.868235:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763847.868239:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763847.868243:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763847.868245:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763847.868247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763847.868250:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763847.868255:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763847.868257:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763847.868260:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763847.868312:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763847.868316:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763847.868319:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763848.252215:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763848.252222:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763848.252224:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763848.252228:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763848.252232:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763848.252236:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763848.252240:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763848.252242:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763848.252246:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763848.892257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763848.892266:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763848.892290:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763848.892291:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763848.892294:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763848.892298:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763848.892301:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763848.892304:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763848.892306:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763848.892307:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763848.892316:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763848.892320:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763848.892323:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763848.892326:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763848.892330:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763848.892361:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763848.892367:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763848.892370:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763849.276211:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763849.276219:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763849.276221:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763849.276228:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763849.276233:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763849.276238:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763849.276241:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763849.276244:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763849.276247:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763849.916206:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763849.916206:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763849.916212:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763849.916213:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763849.916217:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763849.916221:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763849.916221:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763849.916224:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763849.916226:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763849.916229:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763849.916233:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763849.916233:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763849.916240:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763849.916243:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763849.916248:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763849.916313:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763849.916319:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763849.916322:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763850.300260:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763850.300278:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763850.300280:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763850.300284:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763850.300290:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763850.300294:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763850.300297:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763850.300299:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763850.300303:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763850.300328:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763850.300337:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763850.300339:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763850.300343:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763850.300347:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763850.300355:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21564
00000400:00000200:0.0:1787763850.300364:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763850.300370:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763850.300373:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763850.300376:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763850.300379:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763850.300380:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763850.300382:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763850.300386:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763850.300388:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763850.300390:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763850.300394:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763850.300395:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763850.300397:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763850.300399:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763850.300400:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763850.300401:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763850.300403:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763850.300406:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763850.300409:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763850.300411:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763850.300415:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763850.300419:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763850.300421:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763850.300423:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763850.300426:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763850.300438:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763850.300441:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763850.300445:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763850.300446:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763850.300449:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763850.300452:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763850.300456:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763850.300464:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787763850.300471:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763850.300480:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832665984/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763850.300489:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763850.300494:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763850.300498:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832665984/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763850.300527:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763850.300531:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763850.300535:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763850.300537:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763850.300539:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763850.300543:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763850.300547:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21564
00000400:00000200:0.0:1787763850.300553:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763850.300553:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:0.0:1787763850.300559:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787763850.300560:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1787763850.300565:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787763850.300565:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763850.300574:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000040:3.0:1787763850.300580:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f0c0 x1874599832665984/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763850.300581:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763850.300591:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763850.300605:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763850.300606:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787763850.300609:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763850.300618:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:0.0:1787763850.300619:0:304484:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:3.0:1787763850.300622:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:0.0:1787763850.300623:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
02000000:00000001:3.0:1787763850.300627:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763850.300628:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00100000:3.0:1787763850.300634:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832665984:192.168.203.110@tcp:250:kworker.0
00000020:00000040:0.0:1787763850.300635:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763850.300643:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:3.0:1787763850.300643:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763850.300648:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000001:0.0:1787763850.300652:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:3.0:1787763850.300652:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763850.300659:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:0.0:1787763850.300669:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000010:3.0:1787763850.300671:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613800.
00000100:00000001:0.0:1787763850.300680:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1787763850.300681:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763850.300691:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:3.0:1787763850.300692:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763850.300700:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000010:3.0:1787763850.300701:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000020:00000040:0.0:1787763850.300709:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000400:00000010:3.0:1787763850.300710:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00080000:0.0:1787763850.300722:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000200:3.0:1787763850.300723:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832665984, portal 25
00000100:00000001:0.0:1787763850.300734:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763850.300735:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:0.0:1787763850.300746:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787763850.300746:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049530560 : -115096660021056 : ffff9751f876f0c0)
02000000:00000001:0.0:1787763850.300759:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1787763850.300760:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f0c0 x1874599832665984/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763875 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763850.300770:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763850.300779:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763850.300789:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787763850.300789:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000008:00000001:0.0:1787763850.300795:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763850.300801:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763850.300801:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763850.300807:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000010:0.0:1787763850.300810:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000100:00000200:3.0:1787763850.300814:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832665984, offset 0
00000020:00000040:0.0:1787763850.300816:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763850.300823:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763850.300825:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763850.300833:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1787763850.300833:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
02000000:00000001:0.0:1787763850.300838:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763850.300842:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763850.300848:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000400:00000200:3.0:1787763850.300849:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763850.300858:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763850.300867:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832666112/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787763850.300876:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:0.0:1787763850.300882:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763850.300885:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763850.300888:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:0.0:1787763850.300891:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832666112/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787763850.300899:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:0.0:1787763850.300905:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763850.300909:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00080000:0.0:1787763850.300911:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1787763850.300915:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00080000:0.0:1787763850.300923:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763850.300929:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1787763850.300931:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58438:58438:256:4294967295] 192.168.203.110@tcp LPNI seq info [58438:58438:8:4294967295]
00000100:00080000:0.0:1787763850.300932:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763850.300940:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1787763850.300941:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:0.0:1787763850.300951:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21564
00000400:00000200:0.0:1787763850.300961:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763850.300967:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000400:00000200:0.0:1787763850.300974:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763850.300977:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000200:3.0:1787763850.300984:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787763850.300985:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763850.300992:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000010:3.0:1787763850.300994:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000100:00000040:0.0:1787763850.300995:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763850.301004:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763850.301008:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763850.301012:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000800:00000200:3.0:1787763850.301014:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787763850.301015:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763850.301024:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000200:3.0:1787763850.301025:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000020:00000001:0.0:1787763850.301031:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763850.301034:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:3.1:1787763850.301038:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763850.301044:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:3.1:1787763850.301045:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000100:00000001:0.0:1787763850.301052:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763850.301058:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000100:3.0:1787763850.301059:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763850.301066:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000010:3.0:1787763850.301067:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000020:00000040:0.0:1787763850.301076:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:3.0:1787763850.301079:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:0.0:1787763850.301081:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763850.301087:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763850.301094:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:0.0:1787763850.301095:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763850.301105:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1787763850.301107:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
02000000:00000001:0.0:1787763850.301110:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787763850.301111:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:0.0:1787763850.301113:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763850.301118:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787763850.301119:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000008:00000001:0.0:1787763850.301124:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763850.301127:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787763850.301129:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f0c0 x1874599832665984/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763875 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.301165:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763850.301169:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f0c0 x1874599832665984/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763875 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.301175:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763850.301177:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000010:0.0:1787763850.301182:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000100:00000001:3.0:1787763850.301183:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787763850.301188:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000400:00000200:3.0:1787763850.301196:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787763850.301197:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763850.301200:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763850.301202:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316238580
00000100:00000001:0.0:1787763850.301203:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763850.301208:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763850.301211:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763850.301212:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d408a0.
00000100:00000001:3.0:1787763850.301220:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787763850.301228:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316238580
00000100:00000010:3.0:1787763850.301232:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d402a0.
00000100:00000001:3.0:1787763850.301234:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787763850.301242:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763850.301245:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763850.301247:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000010:0.0:1787763850.301250:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00100000:3.0:1787763850.301256:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763850/real 1787763850]  req@ffff9751f876f0c0 x1874599832665984/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763850.301258:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763850.301286:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832666240/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000001:3.0:1787763850.301287:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763850.301290:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763850.301296:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763850.301303:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000100:00000001:0.0:1787763850.301308:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:3.0:1787763850.301308:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:0.0:1787763850.301315:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763850.301317:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:0.0:1787763850.301322:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832666240/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763850.301326:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f0c0 x1874599832665984/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763850.301358:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763850.301363:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:3.0:1787763850.301364:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f0c0 x1874599832665984/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763850.301381:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763850.301390:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763850.301407:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763850.301412:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763850.301417:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763850.301420:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763850.301421:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787763850.301426:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787763850.301430:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21564
00000100:00000001:3.0:1787763850.301431:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763850.301434:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1787763850.301441:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00100000:3.0:1787763850.301446:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f0c0 x1874599832665984/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787763850.301450:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763850.301460:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763850.301472:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763850.301473:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763850.301482:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763850.301494:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000040:3.0:1787763850.301495:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f0c0 x1874599832665984/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763850.301506:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763850.301515:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763850.301518:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000100:00000001:3.0:1787763850.301518:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000001:0.0:1787763850.301529:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787763850.301529:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763850.301536:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000040:0.0:1787763850.301537:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000001:3.0:1787763850.301541:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1787763850.301543:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763850.301550:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:3.0:1787763850.301550:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763850.301559:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:0.0:1787763850.301565:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00080000:3.0:1787763850.301567:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:0.0:1787763850.301568:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763850.301582:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00080000:3.0:1787763850.301584:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763850.301596:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787763850.301596:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000020:00000040:0.0:1787763850.301615:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000040:3.0:1787763850.301619:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f0c0 x1874599832665984/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787763850.301624:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763850.301637:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763850.301649:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763850.301654:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:3.0:1787763850.301654:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832665984:192.168.203.110@tcp:250:kworker.0
02000000:00000001:0.0:1787763850.301665:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763850.301667:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763850.301669:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787763850.301672:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000008:00100000:0.0:1787763850.301675:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000040:3.0:1787763850.301677:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f0c0 x1874599832665984/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:0.0:1787763850.301687:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763850.301694:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763850.301701:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
00000100:00000010:0.0:1787763850.301704:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
02000000:00000001:3.0:1787763850.301704:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000020:00000040:0.0:1787763850.301714:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000010:3.0:1787763850.301715:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613800.
02000000:00000001:3.0:1787763850.301727:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
02000000:00000001:0.0:1787763850.301728:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763850.301735:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787763850.301737:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000100:00000001:0.0:1787763850.301741:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000020:00000040:3.0:1787763850.301742:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763850.301747:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000020:00000001:3.0:1787763850.301748:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000001:0.0:1787763850.301751:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763850.301752:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000010:0.0:1787763850.301761:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
02000000:00000001:3.0:1787763850.301761:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763850.301765:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000001:0.0:1787763850.301768:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763850.301769:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f0c0.
00000100:00000001:3.0:1787763850.301781:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00100000:0.0:1787763850.301785:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832666368/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787763850.301786:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763850.301798:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763850.301808:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763850.301810:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763850.301811:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763850.301815:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763850.301819:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832666368/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.301821:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763850.301826:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763850.301829:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763850.301842:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832666112/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763850.301844:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763850.301855:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21575)
00000100:00000001:3.0:1787763850.301859:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763850.301863:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763850.301868:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763850.301871:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763850.301874:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763850.301878:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832666112:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763850.301882:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763850.301885:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763850.301887:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763850.301889:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763850.301893:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613800.
02000000:00000001:3.0:1787763850.301894:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763850.301898:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763850.301900:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763850.301903:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763850.301905:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832666112, portal 10
00000100:00000001:3.0:1787763850.301907:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763850.301908:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00000040:3.0:1787763850.301914:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832666112/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763875 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.301918:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763850.301920:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763850.301921:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763850.301923:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832666112, offset 0
00000400:00000200:3.0:1787763850.301926:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763850.301929:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763850.301932:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763850.301934:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763850.301936:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763850.301938:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763850.301939:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763850.301941:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58439:58439:256:4294967295] 192.168.203.110@tcp LPNI seq info [58439:58439:8:4294967295]
00000400:00000200:3.0:1787763850.301944:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763850.301949:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763850.301952:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763850.301955:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763850.301957:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763850.301959:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787763850.301963:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763850.301965:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1787763850.301968:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763850.301969:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763850.301972:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763850.301974:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763850.301976:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763850.301977:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763850.301979:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763850.301981:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e580 x1874599832666112/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763875 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.301991:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763850.301993:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832666112/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763875 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.301998:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763850.301999:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763850.302001:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763850.302003:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787763850.302005:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763850.302007:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316237344
00000100:00000001:3.0:1787763850.302009:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316238580
00000100:00000001:3.0:1787763850.302010:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316237344
00000100:00000001:3.0:1787763850.302012:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316238580
00000100:00000001:3.0:1787763850.302014:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763850.302015:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763850.302016:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763850.302019:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763850/real 1787763850]  req@ffff9751f876e580 x1874599832666112/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763850.302023:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763850.302024:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763850.302025:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763850.302027:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763850.302028:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763850.302029:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763850.302031:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832666112/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763850.302036:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832666112/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.302040:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763850.302041:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763850.302043:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763850.302044:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763850.302046:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832666112/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.302050:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763850.302052:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832666112/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.302057:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763850.302058:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763850.302059:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763850.302060:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763850.302062:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763850.302071:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763850.302073:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763850.302075:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763850.302077:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763850.302079:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832666112/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763850.302084:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832666112:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763850.302086:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763850.302089:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832666240/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.302093:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763850.302094:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763850.302095:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763850.302097:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763850.302098:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763850.302100:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832666240:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763850.302102:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763850.302104:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763850.302105:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763850.302106:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763850.302109:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611800.
02000000:00000001:3.0:1787763850.302110:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763850.302111:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763850.302113:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763850.302115:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763850.302117:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832666240, portal 10
00000100:00000001:3.0:1787763850.302118:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763850.302119:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000100:00000040:3.0:1787763850.302123:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832666240/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763875 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.302126:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763850.302128:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763850.302129:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763850.302130:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832666240, offset 0
00000400:00000200:3.0:1787763850.302154:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763850.302157:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763850.302159:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763850.302162:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763850.302163:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763850.302165:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763850.302167:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763850.302169:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58440:58440:256:4294967295] 192.168.203.110@tcp LPNI seq info [58440:58440:8:4294967295]
00000400:00000200:3.0:1787763850.302171:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763850.302175:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763850.302178:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763850.302180:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763850.302183:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763850.302186:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787763850.302188:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763850.302190:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1787763850.302192:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763850.302194:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763850.302196:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763850.302198:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763850.302200:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763850.302201:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763850.302203:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763850.302205:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e1c0 x1874599832666240/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763875 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.302210:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763850.302212:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832666240/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763875 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.302217:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763850.302218:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763850.302220:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763850.302222:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787763850.302224:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763850.302225:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316237333
00000100:00000001:3.0:1787763850.302227:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316238580
00000100:00000001:3.0:1787763850.302228:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316237333
00000100:00000001:3.0:1787763850.302230:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316238580
00000100:00000001:3.0:1787763850.302231:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763850.302232:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763850.302234:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763850.302236:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763850/real 1787763850]  req@ffff9751f876e1c0 x1874599832666240/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763850.302240:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763850.302242:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763850.302243:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763850.302244:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763850.302245:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763850.302247:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763850.302249:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832666240/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763850.302253:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832666240/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.302257:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763850.302258:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763850.302259:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763850.302260:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763850.302279:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832666240/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.302284:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763850.302287:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832666240/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.302291:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763850.302293:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763850.302294:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763850.302295:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763850.302296:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763850.302297:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763850.302298:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763850.302300:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763850.302302:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763850.302304:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832666240/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763850.302313:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832666240:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763850.302316:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763850.302317:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763850.302319:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832666112/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.302323:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763850.302324:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763850.302325:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613800.
02000000:00000001:3.0:1787763850.302327:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763850.302328:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763850.302329:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763850.302330:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763850.302331:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
02000000:00000001:3.0:1787763850.302333:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763850.302334:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763850.302335:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:3.0:1787763850.302338:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763850.302339:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763850.302340:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763850.302342:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832666240/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.302346:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763850.302347:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763850.302348:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611800.
02000000:00000001:3.0:1787763850.302350:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763850.302350:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763850.302351:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763850.302352:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763850.302353:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016c00.
02000000:00000001:3.0:1787763850.302354:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763850.302355:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763850.302356:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:3.0:1787763850.302357:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763850.302358:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763850.302360:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763850.302362:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763850.302363:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763850.302365:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763850.302366:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763850.302367:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763850.302369:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832666368/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.302374:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763850.302375:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763850.302377:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763850.302378:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763850.302379:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763850.302381:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832666368:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763850.302383:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763850.302385:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763850.302386:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763850.302387:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763850.302389:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611800.
02000000:00000001:3.0:1787763850.302391:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763850.302393:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763850.302395:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763850.302397:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763850.302400:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832666368, portal 4
00000100:00000001:3.0:1787763850.302401:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763850.302402:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
00000100:00000040:3.0:1787763850.302406:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832666368/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763875 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.302410:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763850.302412:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763850.302413:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763850.302415:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832666368, offset 0
00000400:00000200:3.0:1787763850.302417:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763850.302420:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763850.302422:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763850.302424:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763850.302426:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763850.302428:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763850.302430:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763850.302432:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58441:58441:256:4294967295] 192.168.203.110@tcp LPNI seq info [58441:58441:8:4294967295]
00000400:00000200:3.0:1787763850.302434:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763850.302439:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763850.302441:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763850.302443:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763850.302446:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763850.302448:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787763850.302450:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763850.302452:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1787763850.302455:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763850.302456:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763850.302458:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763850.302460:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763850.302462:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763850.302463:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763850.302465:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763850.302467:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876c000 x1874599832666368/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763875 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.302472:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763850.302474:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832666368/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763875 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.302478:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763850.302479:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763850.302481:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763850.302483:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787763850.302484:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763850.302486:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316237350
00000100:00000001:3.0:1787763850.302488:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316238580
00000100:00000001:3.0:1787763850.302490:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316237350
00000100:00000001:3.0:1787763850.302491:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316238580
00000100:00000001:3.0:1787763850.302492:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763850.302494:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763850.302495:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763850.302497:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763850/real 1787763850]  req@ffff9751f876c000 x1874599832666368/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763850.302501:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763850.302502:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763850.302503:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763850.302505:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763850.302506:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763850.302507:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763850.302509:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832666368/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763850.302514:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832666368/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.302518:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763850.302519:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763850.302520:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763850.302520:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763850.302523:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832666368/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.302526:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763850.302529:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832666368/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.302533:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763850.302535:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763850.302536:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763850.302537:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763850.302538:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763850.302539:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763850.302541:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763850.302543:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763850.302544:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763850.302546:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832666368/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763850.302551:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832666368:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763850.302553:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763850.302554:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763850.302556:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832666368/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763875 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763850.302559:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763850.302560:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763850.302561:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611800.
02000000:00000001:3.0:1787763850.302563:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763850.302563:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763850.302564:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787763850.302565:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763850.302566:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017800.
02000000:00000001:3.0:1787763850.302568:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763850.302569:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763850.302570:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:3.0:1787763850.302572:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763850.302573:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763850.302574:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763850.302576:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763850.302577:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763850.302579:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763850.302580:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763850.940215:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763850.940221:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763850.940228:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763850.940230:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763850.940233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763850.940234:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763850.940243:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763850.940251:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763850.940256:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763850.940274:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763850.940278:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763850.940282:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763850.940286:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763850.940288:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763850.940292:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763850.940304:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763850.940310:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763850.940314:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763851.324234:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763851.324241:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763851.324242:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763851.324248:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763851.324252:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763851.324266:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763851.324268:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763851.324271:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763851.324274:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763851.580275:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316238580, at 4316238592
00000100:00000040:2.0:1787763851.580291:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316238580, at 4316238592
00000100:00000010:3.0:1787763851.580300:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d402a0.
00000100:00000010:2.0:1787763851.580304:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d408a0.
00000100:00000001:2.0:1787763851.964195:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763851.964202:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763851.964205:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763851.964210:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763851.964213:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763851.964216:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763851.964217:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763851.964220:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763851.964221:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763851.964226:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763851.964227:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763851.964229:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763851.964230:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763851.964235:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763851.964238:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763851.964240:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763851.964243:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763851.964245:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763851.964248:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763851.964262:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763851.964266:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763851.964289:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763851.964294:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763851.964297:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763852.348211:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763852.348218:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763852.348219:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763852.348223:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763852.348226:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763852.348229:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763852.348231:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763852.348233:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763852.348234:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763852.988227:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763852.988233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763852.988235:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763852.988239:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763852.988239:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763852.988260:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763852.988264:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763852.988265:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763852.988269:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763852.988270:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763852.988275:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763852.988277:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763852.988279:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763852.988280:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763852.988284:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763852.988346:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763852.988349:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763852.988351:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763853.372267:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763853.372274:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763853.372275:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763853.372279:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763853.372282:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763853.372285:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763853.372287:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763853.372289:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763853.372291:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763854.012213:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763854.012218:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763854.012220:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763854.012223:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787763854.012227:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763854.012227:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763854.012231:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763854.012233:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00100000:2.0:1787763854.012250:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763854.012253:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763854.012257:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763854.012258:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763854.012262:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763854.012264:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763854.012266:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763854.012269:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763854.012269:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763854.012273:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763854.012274:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763854.012280:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763854.012285:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763854.012331:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763854.012335:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763854.012338:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763854.396262:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763854.396277:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763854.396280:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763854.396286:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763854.396290:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763854.396294:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763854.396297:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763854.396300:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763854.396303:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763855.036212:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763855.036213:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763855.036220:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763855.036220:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763855.036223:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763855.036227:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763855.036228:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763855.036241:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763855.036244:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763855.036245:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763855.036248:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763855.036249:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763855.036254:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763855.036256:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763855.036258:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763855.036293:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763855.036298:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763855.036302:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763855.420218:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:0.0:1787763855.420221:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763855.420240:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1787763855.420243:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787763855.420243:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763855.420246:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:2.0:1787763855.420248:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:0.0:1787763855.420249:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763855.420253:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763855.420253:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763855.420257:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763855.420260:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:0.0:1787763855.420261:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21570
00000100:00000001:2.0:1787763855.420263:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763855.420267:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763855.420269:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763855.420277:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763855.420281:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763855.420286:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763855.420289:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763855.420292:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763855.420295:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763855.420299:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763855.420301:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763855.420304:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763855.420308:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763855.420310:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763855.420313:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763855.420316:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763855.420319:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763855.420320:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763855.420322:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763855.420325:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763855.420327:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763855.420329:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763855.420331:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763855.420334:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763855.420336:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763855.420338:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763855.420340:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763855.420348:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000020:00000040:0.0:1787763855.420350:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763855.420354:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763855.420355:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763855.420357:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763855.420359:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763855.420362:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763855.420369:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787763855.420375:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763855.420382:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832666496/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763855.420390:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763855.420393:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763855.420396:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832666496/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763855.420418:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763855.420420:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763855.420423:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763855.420424:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763855.420426:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763855.420428:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763855.420431:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21570
00000400:00000200:0.0:1787763855.420434:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763855.420437:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763855.420439:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787763855.420439:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763855.420444:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:0.0:1787763855.420445:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763855.420448:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787763855.420448:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763855.420452:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763855.420456:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:3.0:1787763855.420458:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cb40 x1874599832666496/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787763855.420459:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763855.420461:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763855.420464:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763855.420466:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763855.420471:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:3.0:1787763855.420475:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:0.0:1787763855.420476:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:3.0:1787763855.420478:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763855.420482:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:3.0:1787763855.420484:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1787763855.420485:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1787763855.420487:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1787763855.420488:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763855.420493:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:3.0:1787763855.420493:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787763855.420498:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00100000:3.0:1787763855.420499:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832666496:192.168.203.110@tcp:250:kworker.0
00000100:00080000:0.0:1787763855.420503:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763855.420505:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1787763855.420506:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763855.420509:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763855.420510:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763855.420513:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763855.420514:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763855.420515:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1787763855.420519:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763855.420520:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000010:3.0:1787763855.420521:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611800.
00000008:00100000:0.0:1787763855.420524:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1787763855.420525:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1787763855.420530:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763855.420532:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763855.420536:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000010:0.0:1787763855.420539:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
00000400:00000010:3.0:1787763855.420541:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000020:00000040:0.0:1787763855.420542:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763855.420546:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763855.420547:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787763855.420548:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000001:0.0:1787763855.420549:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763855.420552:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763855.420554:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787763855.420557:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832666496, portal 25
02000000:00000010:0.0:1787763855.420558:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:3.0:1787763855.420560:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763855.420563:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049520960 : -115096660030656 : ffff9751f876cb40)
00000100:00000001:0.0:1787763855.420564:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763855.420569:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cb40 x1874599832666496/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763880 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763855.420572:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832666624/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787763855.420583:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1787763855.420587:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:3.0:1787763855.420588:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763855.420591:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787763855.420593:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000040:0.0:1787763855.420597:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832666624/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763855.420597:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832666496, offset 0
00000400:00000200:3.0:1787763855.420607:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763855.420609:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763855.420611:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763855.420614:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763855.420616:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1787763855.420617:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:0.0:1787763855.420619:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763855.420622:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1787763855.420624:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00080000:0.0:1787763855.420625:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21570
00000400:00000200:3.0:1787763855.420631:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763855.420632:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763855.420637:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1787763855.420639:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00080000:0.0:1787763855.420640:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1787763855.420644:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000040:0.0:1787763855.420646:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:3.0:1787763855.420649:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:0.0:1787763855.420650:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763855.420657:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:3.0:1787763855.420657:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58442:58442:256:4294967295] 192.168.203.110@tcp LPNI seq info [58442:58442:8:4294967295]
00000100:00000001:0.0:1787763855.420661:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:3.0:1787763855.420666:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:0.0:1787763855.420667:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763855.420670:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763855.420671:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763855.420673:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763855.420678:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:3.0:1787763855.420678:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000020:00000001:0.0:1787763855.420682:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763855.420687:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:3.0:1787763855.420688:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763855.420690:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763855.420693:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000010:3.0:1787763855.420695:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000100:00000001:0.0:1787763855.420696:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763855.420699:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763855.420704:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000800:00000200:3.0:1787763855.420705:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763855.420707:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763855.420710:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:3.0:1787763855.420710:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
02000000:00000001:0.0:1787763855.420714:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763855.420717:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763855.420718:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763855.420720:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:3.1:1787763855.420722:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000008:00100000:0.0:1787763855.420723:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763855.420729:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1787763855.420730:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763855.420732:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1787763855.420734:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000100:00000010:0.0:1787763855.420737:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000400:00000200:3.0:1787763855.420740:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:0.0:1787763855.420741:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763855.420745:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:3.0:1787763855.420747:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:0.0:1787763855.420748:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763855.420752:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763855.420753:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1787763855.420753:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
02000000:00000001:0.0:1787763855.420755:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787763855.420756:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
02000000:00000010:0.0:1787763855.420759:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:3.0:1787763855.420761:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1787763855.420763:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787763855.420767:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cb40 x1874599832666496/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763880 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763855.420770:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832666752/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787763855.420786:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1787763855.420787:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763855.420789:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763855.420792:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cb40 x1874599832666496/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763880 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763855.420796:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832666752/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.420804:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763855.420807:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763855.420812:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763855.420813:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763855.420814:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763855.420820:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:3.0:1787763855.420822:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:0.0:1787763855.420824:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787763855.420825:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316243699
00000100:00080000:0.0:1787763855.420828:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763855.420831:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763855.420832:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000010:3.0:1787763855.420832:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d402a0.
00000100:00000001:0.0:1787763855.420834:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763855.420838:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21570
00000100:00000001:3.0:1787763855.420840:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000400:00000200:0.0:1787763855.420841:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763855.420846:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763855.420848:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787763855.420849:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316243699
00000100:00000040:0.0:1787763855.420851:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000010:3.0:1787763855.420853:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40d80.
00000100:00000001:0.0:1787763855.420855:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787763855.420857:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000040:0.0:1787763855.420859:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763855.420861:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763855.420862:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787763855.420865:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763855.420866:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763855.420869:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000100:00000001:3.0:1787763855.420869:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000020:00000001:0.0:1787763855.420872:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763855.420875:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00100000:3.0:1787763855.420876:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763855/real 1787763855]  req@ffff9751f876cb40 x1874599832666496/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787763855.420878:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763855.420883:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763855.420890:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763855.420893:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000001:3.0:1787763855.420893:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763855.420895:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:0.0:1787763855.420896:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000010:3.0:1787763855.420898:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000100:00000001:0.0:1787763855.420900:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:3.0:1787763855.420901:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000020:00000040:0.0:1787763855.420903:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000400:00000010:3.0:1787763855.420904:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00080000:0.0:1787763855.420909:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763855.420909:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:0.0:1787763855.420913:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787763855.420914:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cb40 x1874599832666496/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763855.420916:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763855.420919:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763855.420924:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763855.420928:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763855.420932:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000200:3.0:1787763855.420932:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cb40 x1874599832666496/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:0.0:1787763855.420938:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763855.420942:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763855.420948:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763855.420949:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:0.0:1787763855.420950:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000100:00000001:3.0:1787763855.420951:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000040:0.0:1787763855.420953:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:3.0:1787763855.420954:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787763855.420958:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00100000:3.0:1787763855.420960:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cb40 x1874599832666496/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763855.420961:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763855.420965:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1787763855.420969:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787763855.420974:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:3.0:1787763855.420974:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cb40 x1874599832666496/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763855.420977:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763855.420983:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787763855.420987:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763855.420993:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00100000:0.0:1787763855.420996:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832666880/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763855.421001:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763855.421002:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763855.421006:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763855.421009:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832666880/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421010:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763855.421012:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763855.421017:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763855.421020:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763855.421022:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763855.421023:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000040:0.0:1787763855.421027:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21580)
00000100:00080000:3.0:1787763855.421028:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763855.421032:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763855.421036:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cb40 x1874599832666496/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763855.421043:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832666496:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787763855.421046:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763855.421048:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763855.421049:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cb40 x1874599832666496/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421054:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763855.421055:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763855.421058:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611800.
02000000:00000001:3.0:1787763855.421060:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763855.421061:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763855.421063:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787763855.421064:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763855.421066:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016c00.
02000000:00000001:3.0:1787763855.421069:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763855.421070:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763855.421071:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cb40.
00000100:00000001:3.0:1787763855.421074:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763855.421075:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763855.421076:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763855.421082:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763855.421083:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763855.421084:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763855.421085:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763855.421086:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763855.421089:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876da40 x1874599832666624/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421093:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763855.421094:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763855.421096:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763855.421097:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763855.421098:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763855.421100:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832666624:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763855.421102:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763855.421103:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763855.421104:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763855.421105:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763855.421107:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611800.
02000000:00000001:3.0:1787763855.421109:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763855.421111:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763855.421113:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763855.421115:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763855.421118:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832666624, portal 10
00000100:00000001:3.0:1787763855.421119:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763855.421120:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049524800 : -115096660026816 : ffff9751f876da40)
00000100:00000040:3.0:1787763855.421124:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876da40 x1874599832666624/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763880 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421129:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763855.421130:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763855.421132:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763855.421149:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832666624, offset 0
00000400:00000200:3.0:1787763855.421153:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763855.421155:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763855.421158:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763855.421161:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763855.421162:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763855.421164:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763855.421166:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763855.421168:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58443:58443:256:4294967295] 192.168.203.110@tcp LPNI seq info [58443:58443:8:4294967295]
00000400:00000200:3.0:1787763855.421171:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763855.421175:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763855.421178:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763855.421180:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763855.421183:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763855.421185:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787763855.421188:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763855.421190:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1787763855.421192:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763855.421194:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763855.421196:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763855.421198:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763855.421200:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763855.421201:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763855.421203:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763855.421205:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876da40 x1874599832666624/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763880 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421211:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763855.421213:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876da40 x1874599832666624/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763880 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421218:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763855.421219:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763855.421220:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763855.421223:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787763855.421239:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763855.421242:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316242464
00000100:00000001:3.0:1787763855.421244:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316243699
00000100:00000001:3.0:1787763855.421246:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316242464
00000100:00000001:3.0:1787763855.421247:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316243699
00000100:00000001:3.0:1787763855.421249:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763855.421250:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763855.421251:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763855.421254:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763855/real 1787763855]  req@ffff9751f876da40 x1874599832666624/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763855.421260:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763855.421261:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763855.421262:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763855.421264:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763855.421265:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763855.421266:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763855.421268:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876da40 x1874599832666624/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763855.421274:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876da40 x1874599832666624/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421277:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763855.421278:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763855.421280:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763855.421280:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763855.421283:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876da40 x1874599832666624/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421287:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763855.421289:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876da40 x1874599832666624/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421294:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763855.421296:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763855.421297:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763855.421298:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763855.421299:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763855.421309:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763855.421311:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763855.421313:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763855.421315:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763855.421317:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876da40 x1874599832666624/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763855.421322:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832666624:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763855.421325:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763855.421327:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876de00 x1874599832666752/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421330:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763855.421331:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763855.421333:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763855.421334:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763855.421335:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763855.421337:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832666752:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763855.421339:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763855.421340:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763855.421341:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763855.421343:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763855.421345:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613800.
02000000:00000001:3.0:1787763855.421347:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763855.421349:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763855.421350:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763855.421352:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763855.421354:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832666752, portal 10
00000100:00000001:3.0:1787763855.421356:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763855.421357:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049525760 : -115096660025856 : ffff9751f876de00)
00000100:00000040:3.0:1787763855.421361:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876de00 x1874599832666752/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763880 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421365:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763855.421366:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763855.421368:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763855.421369:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832666752, offset 0
00000400:00000200:3.0:1787763855.421372:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763855.421374:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763855.421376:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763855.421378:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763855.421380:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763855.421381:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763855.421383:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763855.421385:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58444:58444:256:4294967295] 192.168.203.110@tcp LPNI seq info [58444:58444:8:4294967295]
00000400:00000200:3.0:1787763855.421388:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763855.421392:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763855.421395:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763855.421397:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763855.421399:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763855.421401:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787763855.421404:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763855.421406:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1787763855.421408:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763855.421410:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763855.421412:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763855.421414:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763855.421416:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763855.421417:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763855.421418:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763855.421420:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876de00 x1874599832666752/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763880 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421425:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763855.421428:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876de00 x1874599832666752/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763880 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421432:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763855.421433:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763855.421435:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763855.421437:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787763855.421438:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763855.421439:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316242452
00000100:00000001:3.0:1787763855.421441:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316243699
00000100:00000001:3.0:1787763855.421443:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316242452
00000100:00000001:3.0:1787763855.421444:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316243699
00000100:00000001:3.0:1787763855.421446:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763855.421447:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763855.421448:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763855.421450:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763855/real 1787763855]  req@ffff9751f876de00 x1874599832666752/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763855.421454:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763855.421455:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763855.421456:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763855.421458:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763855.421459:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763855.421460:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763855.421462:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876de00 x1874599832666752/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763855.421467:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876de00 x1874599832666752/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421471:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763855.421471:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763855.421472:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763855.421473:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763855.421476:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876de00 x1874599832666752/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421480:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763855.421483:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876de00 x1874599832666752/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421487:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763855.421488:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763855.421489:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763855.421490:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763855.421491:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763855.421492:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763855.421494:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763855.421495:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763855.421497:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763855.421499:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876de00 x1874599832666752/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763855.421503:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832666752:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763855.421505:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763855.421507:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832666880/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421511:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763855.421512:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763855.421513:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763855.421514:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763855.421516:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763855.421517:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832666880:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763855.421520:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763855.421521:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763855.421522:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763855.421523:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763855.421526:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610400.
02000000:00000001:3.0:1787763855.421527:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763855.421529:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763855.421531:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763855.421532:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763855.421534:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832666880, portal 4
00000100:00000001:3.0:1787763855.421535:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763855.421536:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
00000100:00000040:3.0:1787763855.421540:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832666880/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763880 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421544:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763855.421545:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763855.421547:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763855.421548:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832666880, offset 0
00000400:00000200:3.0:1787763855.421550:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763855.421552:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763855.421554:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763855.421556:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763855.421558:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763855.421559:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763855.421561:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763855.421563:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58445:58445:256:4294967295] 192.168.203.110@tcp LPNI seq info [58445:58445:8:4294967295]
00000400:00000200:3.0:1787763855.421565:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763855.421569:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763855.421571:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763855.421573:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763855.421575:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763855.421576:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787763855.421579:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763855.421581:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1787763855.421583:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763855.421585:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763855.421587:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763855.421589:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763855.421591:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763855.421592:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763855.421593:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763855.421595:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876c000 x1874599832666880/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763880 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421600:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763855.421602:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832666880/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763880 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421606:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763855.421607:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763855.421609:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763855.421611:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787763855.421612:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763855.421614:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316242469
00000100:00000001:3.0:1787763855.421615:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316243699
00000100:00000001:3.0:1787763855.421617:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316242469
00000100:00000001:3.0:1787763855.421618:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316243699
00000100:00000001:3.0:1787763855.421619:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763855.421620:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763855.421622:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763855.421624:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763855/real 1787763855]  req@ffff9751f876c000 x1874599832666880/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763855.421628:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763855.421629:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763855.421630:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763855.421631:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763855.421632:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763855.421633:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763855.421635:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832666880/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763855.421641:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832666880/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421644:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763855.421645:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763855.421646:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763855.421647:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763855.421649:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832666880/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421653:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763855.421655:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832666880/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421659:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763855.421660:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763855.421661:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763855.421662:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763855.421663:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763855.421664:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763855.421665:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763855.421667:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763855.421668:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763855.421671:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832666880/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763855.421676:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832666880:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763855.421678:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763855.421679:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763855.421682:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876da40 x1874599832666624/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421686:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763855.421687:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763855.421688:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611800.
02000000:00000001:3.0:1787763855.421690:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763855.421691:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763855.421691:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763855.421693:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763855.421694:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
02000000:00000001:3.0:1787763855.421695:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763855.421696:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763855.421697:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876da40.
00000100:00000001:3.0:1787763855.421699:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763855.421700:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763855.421701:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763855.421703:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876de00 x1874599832666752/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421707:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763855.421708:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763855.421709:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613800.
02000000:00000001:3.0:1787763855.421711:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763855.421711:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763855.421712:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763855.421713:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763855.421714:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:3.0:1787763855.421715:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763855.421716:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763855.421717:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876de00.
00000100:00000001:3.0:1787763855.421719:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763855.421720:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763855.421720:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763855.421722:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832666880/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763880 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763855.421726:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763855.421727:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763855.421728:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610400.
02000000:00000001:3.0:1787763855.421729:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763855.421730:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763855.421731:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787763855.421732:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763855.421733:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014400.
02000000:00000001:3.0:1787763855.421734:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763855.421735:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763855.421736:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:3.0:1787763855.421737:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763855.421738:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763855.421739:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763855.421741:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763855.421742:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763855.421743:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763855.421744:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763856.060199:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763856.060208:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763856.060212:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763856.060218:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763856.060234:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763856.060238:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763856.060240:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763856.060243:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763856.060246:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763856.060255:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763856.060261:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763856.060266:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763856.060271:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763856.060274:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763856.060277:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763856.060281:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763856.060283:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763856.060285:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763856.060287:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763856.060291:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763856.060292:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763856.060347:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763856.060351:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763856.060354:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763856.444296:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763856.444312:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763856.444315:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763856.444321:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763856.444325:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763856.444329:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763856.444332:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763856.444335:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763856.444338:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763856.700206:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316243699, at 4316243712
00000100:00000010:3.0:1787763856.700262:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40d80.
00000100:00000040:2.0:1787763856.700336:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316243699, at 4316243712
00000100:00000010:2.0:1787763856.700353:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d402a0.
00000100:00000001:2.0:1787763857.084254:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763857.084259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763857.084262:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763857.084268:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763857.084268:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763857.084281:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763857.084283:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763857.084290:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763857.084291:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763857.084295:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763857.084297:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763857.084299:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763857.084303:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763857.084306:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763857.084308:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763857.084358:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763857.084363:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763857.084366:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763857.468208:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763857.468226:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763857.468228:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763857.468233:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763857.468238:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763857.468241:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763857.468245:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763857.468247:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763857.468249:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763858.108290:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763858.108298:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763858.108301:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763858.108306:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763858.108309:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763858.108309:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763858.108315:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763858.108319:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763858.108321:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763858.108325:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763858.108326:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763858.108330:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763858.108333:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763858.108335:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763858.108338:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763858.108380:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763858.108384:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763858.108388:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763858.492255:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763858.492262:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763858.492264:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763858.492269:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763858.492274:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763858.492277:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763858.492280:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763858.492283:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763858.492286:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763859.132258:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763859.132266:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763859.132267:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763859.132270:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763859.132274:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763859.132278:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763859.132284:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763859.132286:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763859.132288:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763859.132292:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763859.132296:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763859.132298:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763859.132300:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763859.132304:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763859.132309:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763859.132387:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763859.132394:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763859.132398:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763859.516225:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763859.516233:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763859.516235:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763859.516239:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763859.516242:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763859.516246:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763859.516248:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763859.516251:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763859.516253:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763860.156244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763860.156250:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763860.156254:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763860.156258:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763860.156262:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763860.156264:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763860.156268:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763860.156269:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763860.156273:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763860.156278:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763860.156278:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763860.156286:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763860.156287:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763860.156292:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763860.156296:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763860.156359:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763860.156365:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763860.156370:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763860.540257:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:0.0:1787763860.540261:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763860.540264:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763860.540270:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763860.540283:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00100000:2.0:1787763860.540283:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763860.540292:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787763860.540293:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1787763860.540297:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763860.540299:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763860.540308:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763860.540311:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763860.540312:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787763860.540323:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21575
00000100:00000001:2.0:1787763860.540325:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763860.540342:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763860.540353:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763860.540356:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763860.540359:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763860.540362:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763860.540364:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763860.540367:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763860.540371:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763860.540373:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763860.540376:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763860.540380:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763860.540381:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763860.540383:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763860.540389:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763860.540390:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763860.540392:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763860.540394:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763860.540396:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763860.540399:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763860.540401:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763860.540403:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763860.540407:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763860.540409:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763860.540411:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763860.540413:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763860.540423:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763860.540425:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763860.540429:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763860.540430:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763860.540434:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763860.540437:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763860.540441:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763860.540449:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763860.540456:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763860.540464:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832667008/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763860.540478:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763860.540482:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763860.540487:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832667008/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763860.540505:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763860.540507:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763860.540510:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763860.540511:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763860.540513:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763860.540516:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763860.540519:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21575
00000400:00000200:0.0:1787763860.540522:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763860.540524:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763860.540526:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763860.540528:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763860.540532:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787763860.540532:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1787763860.540535:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763860.540540:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787763860.540540:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763860.540545:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1787763860.540546:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763860.540549:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763860.540553:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763860.540558:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000040:3.0:1787763860.540560:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832667008/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787763860.540563:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763860.540569:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:3.0:1787763860.540577:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763860.540579:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763860.540583:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:0.0:1787763860.540585:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:3.0:1787763860.540586:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763860.540591:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1787763860.540591:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763860.540596:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00100000:3.0:1787763860.540598:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832667008:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763860.540599:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763860.540606:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00000001:3.0:1787763860.540606:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:0.0:1787763860.540611:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763860.540613:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763860.540613:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763860.540616:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763860.540617:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763860.540619:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763860.540626:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610400.
02000000:00000001:3.0:1787763860.540627:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763860.540629:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763860.540632:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763860.540634:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763860.540635:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:3.0:1787763860.540641:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000008:00100000:0.0:1787763860.540643:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763860.540647:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763860.540649:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787763860.540650:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000010:0.0:1787763860.540654:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763860.540657:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000100:00000200:3.0:1787763860.540659:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832667008, portal 25
02000000:00000001:0.0:1787763860.540660:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1787763860.540663:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:0.0:1787763860.540664:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763860.540667:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000100:00000001:0.0:1787763860.540668:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763860.540671:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763860.540673:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763860.540676:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832667008/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763885 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1787763860.540679:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763860.540685:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763860.540686:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763860.540690:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00100000:0.0:1787763860.540693:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832667136/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1787763860.540695:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763860.540700:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832667008, offset 0
00000100:00000001:0.0:1787763860.540707:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763860.540710:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763860.540711:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787763860.540715:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832667136/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787763860.540722:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763860.540723:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763860.540729:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1787763860.540732:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00080000:0.0:1787763860.540738:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:3.0:1787763860.540740:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1787763860.540742:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763860.540747:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1787763860.540747:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763860.540752:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1787763860.540754:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1787763860.540756:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00080000:0.0:1787763860.540764:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21575
00000400:00000200:3.0:1787763860.540764:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58446:58446:256:4294967295] 192.168.203.110@tcp LPNI seq info [58446:58446:8:4294967295]
00000400:00000200:0.0:1787763860.540772:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763860.540778:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763860.540782:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1787763860.540784:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:0.0:1787763860.540788:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763860.540791:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763860.540797:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763860.540801:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:3.0:1787763860.540801:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000020:00000001:0.0:1787763860.540806:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763860.540809:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763860.540811:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:3.0:1787763860.540812:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000020:00000040:0.0:1787763860.540815:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763860.540817:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763860.540820:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000010:3.0:1787763860.540820:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000100:00000040:0.0:1787763860.540826:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763860.540827:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1787763860.540831:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763860.540832:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763860.540836:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:3.0:1787763860.540838:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000020:00000040:0.0:1787763860.540839:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763860.540844:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763860.540847:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763860.540848:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:0.0:1787763860.540849:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763860.540854:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:3.1:1787763860.540854:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
02000000:00000001:0.0:1787763860.540861:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1787763860.540864:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763860.540865:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000010:3.0:1787763860.540868:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000008:00100000:0.0:1787763860.540869:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763860.540873:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763860.540875:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763860.540877:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000010:0.0:1787763860.540878:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763860.540882:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000400:00000200:3.0:1787763860.540884:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:0.0:1787763860.540886:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763860.540888:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763860.540890:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1787763860.540890:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
02000000:00000001:0.0:1787763860.540893:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:3.0:1787763860.540894:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
02000000:00000001:0.0:1787763860.540896:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763860.540901:0:257575:0:(events.c:34:request_out_callback()) Process entered
02000000:00000010:0.0:1787763860.540902:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000200:3.0:1787763860.540907:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e1c0 x1874599832667008/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763885 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763860.540908:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763860.540917:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832667264/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787763860.540921:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1787763860.540927:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:3.0:1787763860.540928:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832667008/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763885 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763860.540933:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763860.540937:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763860.540939:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:0.0:1787763860.540943:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832667264/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.540945:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763860.540953:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763860.540958:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316248819
00000100:00000001:0.0:1787763860.540965:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763860.540966:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40d80.
00000100:00000040:0.0:1787763860.540970:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763860.540975:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1787763860.540976:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00080000:0.0:1787763860.540978:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763860.540982:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763860.540983:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763860.540985:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763860.540986:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316248819
00000100:00000001:0.0:1787763860.540987:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000010:3.0:1787763860.540991:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40a80.
00000100:00080000:0.0:1787763860.540992:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21575
00000100:00000001:3.0:1787763860.540996:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000400:00000200:0.0:1787763860.541000:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763860.541001:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1787763860.541005:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787763860.541006:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763860.541009:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787763860.541011:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000040:0.0:1787763860.541016:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00100000:3.0:1787763860.541019:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763860/real 1787763860]  req@ffff9751f876e1c0 x1874599832667008/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763860.541020:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763860.541025:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763860.541030:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763860.541035:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763860.541037:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000400:00000001:3.0:1787763860.541038:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000001:0.0:1787763860.541039:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000001:3.0:1787763860.541040:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000020:00000040:0.0:1787763860.541043:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000400:00000010:3.0:1787763860.541044:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000020:00000001:0.0:1787763860.541048:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000200:3.0:1787763860.541049:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000020:00000001:0.0:1787763860.541053:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000010:3.0:1787763860.541053:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763860.541058:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:0.0:1787763860.541061:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763860.541065:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:3.0:1787763860.541065:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832667008/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763860.541070:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763860.541074:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763860.541078:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763860.541083:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763860.541087:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763860.541089:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:3.0:1787763860.541092:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832667008/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763860.541093:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763860.541098:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763860.541102:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763860.541105:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763860.541110:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763860.541113:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763860.541117:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763860.541121:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763860.541124:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1787763860.541124:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000001:0.0:1787763860.541128:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763860.541128:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763860.541130:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1787763860.541132:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:0.0:1787763860.541152:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763860.541155:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763860.541158:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763860.541161:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763860.541165:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763860.541166:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832667392/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:3.0:1787763860.541173:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832667008/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763860.541178:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763860.541182:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763860.541188:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763860.541203:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832667392/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787763860.541209:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832667008/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763860.541224:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763860.541226:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763860.541230:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763860.541232:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21585)
00000100:00000001:3.0:1787763860.541232:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763860.541235:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763860.541238:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763860.541240:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763860.541242:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763860.541246:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763860.541248:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763860.541251:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832667008/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763860.541257:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832667008:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787763860.541261:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763860.541262:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763860.541264:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832667008/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541270:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763860.541272:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763860.541275:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610400.
02000000:00000001:3.0:1787763860.541277:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763860.541279:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763860.541281:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787763860.541282:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763860.541285:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017000.
02000000:00000001:3.0:1787763860.541290:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763860.541292:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763860.541293:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:3.0:1787763860.541297:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763860.541298:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763860.541300:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763860.541306:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763860.541307:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763860.541309:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763860.541310:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763860.541311:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763860.541314:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832667136/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541319:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763860.541320:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763860.541321:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763860.541323:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763860.541324:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763860.541326:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832667136:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763860.541328:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763860.541330:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763860.541331:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763860.541332:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763860.541335:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610400.
02000000:00000001:3.0:1787763860.541336:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763860.541339:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763860.541341:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763860.541344:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763860.541347:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832667136, portal 10
00000100:00000001:3.0:1787763860.541349:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763860.541350:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00000040:3.0:1787763860.541355:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832667136/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763885 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541360:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763860.541361:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763860.541363:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763860.541364:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832667136, offset 0
00000400:00000200:3.0:1787763860.541367:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763860.541370:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763860.541373:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763860.541375:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763860.541377:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763860.541379:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763860.541381:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763860.541383:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58447:58447:256:4294967295] 192.168.203.110@tcp LPNI seq info [58447:58447:8:4294967295]
00000400:00000200:3.0:1787763860.541386:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763860.541390:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763860.541394:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763860.541396:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763860.541399:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763860.541401:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787763860.541404:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763860.541406:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1787763860.541409:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763860.541410:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763860.541412:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763860.541415:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763860.541417:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763860.541418:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763860.541420:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763860.541422:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e580 x1874599832667136/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763885 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541432:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763860.541434:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832667136/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763885 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541439:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763860.541440:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763860.541442:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763860.541445:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787763860.541447:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763860.541448:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316247584
00000100:00000001:3.0:1787763860.541450:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316248819
00000100:00000001:3.0:1787763860.541452:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316247584
00000100:00000001:3.0:1787763860.541453:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316248819
00000100:00000001:3.0:1787763860.541455:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763860.541456:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763860.541457:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763860.541460:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763860/real 1787763860]  req@ffff9751f876e580 x1874599832667136/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763860.541464:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763860.541465:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763860.541466:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763860.541468:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763860.541469:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763860.541470:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763860.541473:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832667136/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763860.541478:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832667136/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541481:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763860.541483:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763860.541485:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763860.541486:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763860.541488:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832667136/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541492:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763860.541494:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832667136/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541499:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763860.541500:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763860.541501:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763860.541502:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763860.541503:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763860.541514:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763860.541516:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763860.541519:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763860.541521:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763860.541525:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832667136/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763860.541534:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832667136:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763860.541537:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763860.541541:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f0c0 x1874599832667264/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541548:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763860.541549:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763860.541552:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763860.541554:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763860.541556:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763860.541560:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832667264:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763860.541563:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763860.541565:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763860.541567:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763860.541569:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763860.541572:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613800.
02000000:00000001:3.0:1787763860.541574:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763860.541578:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763860.541581:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763860.541584:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763860.541588:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832667264, portal 10
00000100:00000001:3.0:1787763860.541589:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763860.541590:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049530560 : -115096660021056 : ffff9751f876f0c0)
00000100:00000040:3.0:1787763860.541595:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f0c0 x1874599832667264/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763885 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541600:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763860.541602:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763860.541603:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763860.541605:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832667264, offset 0
00000400:00000200:3.0:1787763860.541607:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763860.541610:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763860.541612:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763860.541614:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763860.541616:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763860.541618:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763860.541620:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763860.541622:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58448:58448:256:4294967295] 192.168.203.110@tcp LPNI seq info [58448:58448:8:4294967295]
00000400:00000200:3.0:1787763860.541625:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763860.541629:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763860.541632:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763860.541635:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763860.541638:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763860.541640:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787763860.541643:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763860.541645:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1787763860.541647:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763860.541649:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763860.541651:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763860.541654:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763860.541656:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763860.541657:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763860.541659:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763860.541661:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f0c0 x1874599832667264/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763885 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541667:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763860.541669:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f0c0 x1874599832667264/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763885 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541674:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763860.541675:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763860.541677:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763860.541679:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787763860.541681:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763860.541682:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316247572
00000100:00000001:3.0:1787763860.541685:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316248819
00000100:00000001:3.0:1787763860.541686:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316247572
00000100:00000001:3.0:1787763860.541688:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316248819
00000100:00000001:3.0:1787763860.541689:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763860.541691:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763860.541692:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763860.541694:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763860/real 1787763860]  req@ffff9751f876f0c0 x1874599832667264/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763860.541699:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763860.541700:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763860.541701:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763860.541703:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763860.541704:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763860.541705:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763860.541707:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f0c0 x1874599832667264/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763860.541712:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f0c0 x1874599832667264/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541716:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763860.541717:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763860.541718:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763860.541719:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763860.541721:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f0c0 x1874599832667264/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541725:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763860.541727:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f0c0 x1874599832667264/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541732:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763860.541733:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763860.541735:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763860.541735:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763860.541737:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763860.541738:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763860.541739:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763860.541741:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763860.541742:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763860.541745:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f0c0 x1874599832667264/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763860.541754:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832667264:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763860.541757:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763860.541759:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832667392/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541763:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763860.541764:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763860.541765:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763860.541766:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763860.541768:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763860.541769:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832667392:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763860.541771:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763860.541772:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763860.541773:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763860.541774:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763860.541777:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611800.
02000000:00000001:3.0:1787763860.541778:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763860.541780:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763860.541782:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763860.541784:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763860.541786:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832667392, portal 4
00000100:00000001:3.0:1787763860.541788:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763860.541789:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
00000100:00000040:3.0:1787763860.541793:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832667392/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763885 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541797:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763860.541798:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763860.541800:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763860.541801:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832667392, offset 0
00000400:00000200:3.0:1787763860.541804:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763860.541806:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763860.541809:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763860.541811:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763860.541812:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763860.541814:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763860.541816:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763860.541818:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58449:58449:256:4294967295] 192.168.203.110@tcp LPNI seq info [58449:58449:8:4294967295]
00000400:00000200:3.0:1787763860.541821:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763860.541825:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763860.541828:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763860.541829:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763860.541832:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763860.541834:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787763860.541836:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763860.541838:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1787763860.541841:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763860.541843:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763860.541845:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763860.541847:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763860.541849:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763860.541850:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763860.541851:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763860.541854:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876ed00 x1874599832667392/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763885 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541859:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763860.541861:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832667392/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763885 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541865:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763860.541866:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763860.541868:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763860.541870:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787763860.541872:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763860.541874:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316247589
00000100:00000001:3.0:1787763860.541875:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316248819
00000100:00000001:3.0:1787763860.541877:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316247589
00000100:00000001:3.0:1787763860.541878:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316248819
00000100:00000001:3.0:1787763860.541880:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763860.541881:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763860.541882:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763860.541884:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763860/real 1787763860]  req@ffff9751f876ed00 x1874599832667392/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763860.541888:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763860.541889:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763860.541890:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763860.541891:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763860.541892:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763860.541894:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763860.541896:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832667392/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763860.541901:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832667392/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541905:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763860.541906:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763860.541907:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763860.541908:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763860.541910:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832667392/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541914:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763860.541916:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832667392/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541920:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763860.541922:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763860.541923:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763860.541924:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763860.541925:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763860.541926:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763860.541928:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763860.541930:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763860.541931:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763860.541933:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832667392/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763860.541938:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832667392:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763860.541940:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763860.541941:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763860.541943:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832667136/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541947:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763860.541947:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763860.541949:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610400.
02000000:00000001:3.0:1787763860.541950:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763860.541951:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763860.541952:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763860.541953:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763860.541954:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014800.
02000000:00000001:3.0:1787763860.541956:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763860.541957:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763860.541958:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:3.0:1787763860.541960:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763860.541961:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763860.541962:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763860.541964:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f0c0 x1874599832667264/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541973:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763860.541974:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763860.541975:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613800.
02000000:00000001:3.0:1787763860.541977:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763860.541977:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763860.541978:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763860.541979:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763860.541980:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:3.0:1787763860.541981:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763860.541982:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763860.541983:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f0c0.
00000100:00000001:3.0:1787763860.541984:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763860.541985:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763860.541986:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763860.541988:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832667392/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763885 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763860.541991:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763860.541992:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763860.541993:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611800.
02000000:00000001:3.0:1787763860.541994:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763860.541995:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763860.541995:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787763860.541996:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763860.541997:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015c00.
02000000:00000001:3.0:1787763860.541998:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763860.541999:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763860.542000:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:3.0:1787763860.542001:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763860.542002:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763860.542003:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763860.542005:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763860.542006:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763860.542007:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763860.542008:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763861.180224:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763861.180231:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763861.180234:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763861.180234:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763861.180240:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763861.180241:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763861.180245:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763861.180246:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763861.180248:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763861.180250:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763861.180253:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763861.180254:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763861.180256:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763861.180257:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763861.180261:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763861.180300:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763861.180304:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763861.180308:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763861.564206:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763861.564213:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763861.564215:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763861.564220:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763861.564224:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763861.564228:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763861.564231:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763861.564235:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763861.564238:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763861.820228:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316248819, at 4316248832
00000100:00000010:3.0:1787763861.820247:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40a80.
00000100:00000040:2.0:1787763861.820251:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316248819, at 4316248832
00000100:00000010:2.0:1787763861.820265:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40d80.
00010000:00000001:3.0:1787763862.204229:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763862.204236:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763862.204241:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763862.204243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763862.204247:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763862.204247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763862.204251:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763862.204253:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763862.204255:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763862.204259:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763862.204260:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763862.204264:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763862.204266:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763862.204268:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763862.204271:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763862.204271:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763862.204277:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763862.204281:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763862.204284:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763862.204286:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763862.204289:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763862.204349:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763862.204354:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763862.204357:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763862.588237:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763862.588246:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763862.588249:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763862.588255:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763862.588259:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763862.588262:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763862.588265:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763862.588267:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763862.588271:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763863.228224:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763863.228226:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763863.228231:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763863.228234:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763863.228238:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763863.228238:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763863.228240:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763863.228245:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763863.228246:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763863.228246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763863.228253:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763863.228257:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763863.228259:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763863.228261:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763863.228265:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763863.228300:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763863.228305:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763863.228309:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763863.612278:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763863.612287:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763863.612290:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763863.612296:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763863.612300:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763863.612305:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763863.612308:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763863.612311:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763863.612314:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763864.252240:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763864.252246:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763864.252248:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763864.252253:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763864.252258:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763864.252260:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763864.252262:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763864.252266:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763864.252269:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763864.252272:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763864.252273:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763864.252279:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763864.252284:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763864.252289:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763864.252292:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763864.252295:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763864.252297:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763864.252302:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763864.252304:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763864.252315:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763864.252317:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763864.252362:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763864.252365:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763864.252368:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763864.636305:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763864.636315:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763864.636318:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763864.636324:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763864.636328:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763864.636334:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763864.636338:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763864.636341:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763864.636345:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763865.276251:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763865.276255:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763865.276260:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763865.276261:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763865.276270:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763865.276272:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763865.276274:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763865.276280:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763865.276281:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763865.276282:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763865.276291:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763865.276295:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763865.276299:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763865.276302:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763865.276306:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763865.276339:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763865.276344:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763865.276347:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763865.660303:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763865.660313:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763865.660317:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763865.660328:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1787763865.660328:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763865.660339:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:1.0:1787763865.660342:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787763865.660348:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763865.660349:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763865.660356:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763865.660358:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763865.660364:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763865.660367:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763865.660371:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763865.660382:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21580
00000400:00000200:1.0:1787763865.660397:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763865.660406:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763865.660411:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763865.660415:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763865.660420:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763865.660423:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763865.660426:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763865.660431:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763865.660434:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787763865.660442:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763865.660446:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787763865.660448:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787763865.660454:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787763865.660457:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763865.660460:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763865.660461:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763865.660464:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763865.660467:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787763865.660472:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763865.660476:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763865.660480:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763865.660486:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763865.660490:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763865.660493:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763865.660496:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763865.660506:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933480.
00000020:00000040:1.0:1787763865.660511:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787763865.660515:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763865.660517:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763865.660521:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763865.660525:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763865.660531:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763865.660540:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9400.
00000100:00000001:1.0:1787763865.660550:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763865.660562:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933480 x1874599832667520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763865.660575:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763865.660582:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763865.660588:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3c2c33c] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933480 x1874599832667520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763865.660617:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763865.660621:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763865.660625:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787763865.660627:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763865.660629:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763865.660632:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763865.660636:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21580
00000400:00000200:1.0:1787763865.660640:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763865.660644:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763865.660647:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763865.660668:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763865.660670:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763865.660673:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763865.660675:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763865.660678:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763865.660680:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787763865.660681:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763865.660684:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787763865.660686:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787763865.660688:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787763865.660691:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763865.660693:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763865.660695:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763865.660701:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763865.660704:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787763865.660707:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763865.660709:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763865.660712:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763865.660718:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763865.660720:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763865.660722:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763865.660729:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763865.660731:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763865.660733:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763865.660738:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930000.
00000020:00000040:1.0:1787763865.660740:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763865.660743:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763865.660744:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763865.660747:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763865.660748:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763865.660750:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763865.660754:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cac00.
00000100:00000001:1.0:1787763865.660759:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763865.660764:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930000 x1874599832667648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763865.660771:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763865.660774:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763865.660778:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6d90274] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930000 x1874599832667648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763865.660787:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763865.660790:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763865.660794:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787763865.660795:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763865.660797:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763865.660800:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763865.660802:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21580
00000400:00000200:1.0:1787763865.660807:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763865.660810:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763865.660813:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763865.660816:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763865.660819:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763865.660821:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763865.660823:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763865.660825:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763865.660826:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787763865.660828:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763865.660830:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787763865.660831:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787763865.660832:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787763865.660835:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763865.660836:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763865.660838:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763865.660840:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763865.660842:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787763865.660844:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763865.660846:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763865.660847:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763865.660849:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763865.660850:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763865.660852:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763865.660853:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763865.660855:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763865.660856:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763865.660860:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930b40.
00000020:00000040:1.0:1787763865.660863:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763865.660866:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763865.660867:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763865.660869:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763865.660871:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763865.660873:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763865.660877:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9c00.
00000100:00000001:1.0:1787763865.660881:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763865.660886:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930b40 x1874599832667776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763865.660894:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763865.660897:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763865.660901:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022d7f91a] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930b40 x1874599832667776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763865.660911:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763865.660914:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787763865.660918:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787763865.660921:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763865.660925:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787763865.660927:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763865.660928:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763865.660931:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763865.660934:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21580
00000400:00000200:1.0:1787763865.660939:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763865.660942:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763865.660945:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763865.660949:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763865.660951:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763865.660954:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763865.660957:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763865.660960:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763865.660962:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787763865.660964:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763865.660966:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787763865.660969:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787763865.660972:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:1.0:1787763865.660975:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763865.660978:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763865.660980:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763865.660983:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763865.660986:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:1.0:1787763865.660989:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763865.660992:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763865.660994:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763865.660999:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763865.661001:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763865.661003:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763865.661006:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1787763865.661010:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763865.661011:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763865.661015:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7931680.
00000020:00000040:1.0:1787763865.661018:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787763865.661021:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763865.661022:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763865.661024:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763865.661026:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763865.661028:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763865.661032:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb000.
00000100:00000001:1.0:1787763865.661036:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763865.661042:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7931680 x1874599832667904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763865.661050:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763865.661052:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763865.661056:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d4255d6] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7931680 x1874599832667904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763865.661064:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763865.661069:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21590)
00000100:00000001:3.0:1787763865.661256:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763865.661264:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763865.661268:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763865.661278:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933480 x1874599832667520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.661290:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763865.661293:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763865.661298:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763865.661302:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763865.661306:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763865.661311:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832667520:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787763865.661317:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763865.661323:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763865.661325:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763865.661328:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763865.661333:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611800.
02000000:00000001:3.0:1787763865.661334:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763865.661341:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763865.661344:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763865.661353:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763865.661360:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832667520, portal 25
00000100:00000001:3.0:1787763865.661362:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763865.661364:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497734784 : -115097211816832 : ffff9751d7933480)
00000100:00000040:3.0:1787763865.661370:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933480 x1874599832667520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763890 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.661376:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763865.661379:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763865.661381:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763865.661384:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832667520, offset 0
00000400:00000200:3.0:1787763865.661391:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763865.661401:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763865.661410:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763865.661417:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763865.661423:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763865.661428:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763865.661431:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763865.661436:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58450:58450:256:4294967295] 192.168.203.110@tcp LPNI seq info [58450:58450:8:4294967295]
00000400:00000200:3.0:1787763865.661441:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763865.661455:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763865.661465:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763865.661481:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763865.661490:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763865.661495:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787763865.661500:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763865.661504:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787763865.661509:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763865.661513:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763865.661518:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763865.661524:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763865.661530:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763865.661534:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763865.661540:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763865.661545:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933480 x1874599832667520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763890 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.661569:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763865.661572:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933480 x1874599832667520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763890 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.661579:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763865.661580:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763865.661587:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763865.661591:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763865.661594:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316253940
00000100:00000010:3.0:1787763865.661599:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40a80.
00000100:00000001:3.0:1787763865.661610:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787763865.661619:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316253940
00000100:00000010:3.0:1787763865.661622:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40960.
00000100:00000001:3.0:1787763865.661626:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787763865.661630:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763865.661633:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763865.661636:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763865.661641:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763865/real 1787763865]  req@ffff9751d7933480 x1874599832667520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763865.661652:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763865.661654:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763865.661661:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763865.661664:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763865.661666:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763865.661670:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763865.661674:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933480 x1874599832667520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763865.661684:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933480 x1874599832667520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.661692:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763865.661694:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763865.661748:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763865.661750:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763865.661754:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933480 x1874599832667520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.661760:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763865.661764:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933480 x1874599832667520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.661774:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763865.661777:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763865.661778:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763865.661779:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763865.661781:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763865.661782:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763865.661784:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763865.661787:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763865.661789:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763865.661791:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933480 x1874599832667520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763865.661797:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832667520:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787763865.661800:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763865.661803:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930000 x1874599832667648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.661808:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763865.661809:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763865.661811:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763865.661812:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763865.661813:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763865.661815:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832667648:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763865.661817:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763865.661819:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763865.661821:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763865.661822:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763865.661825:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613800.
02000000:00000001:3.0:1787763865.661826:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763865.661829:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763865.661832:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763865.661835:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763865.661837:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832667648, portal 10
00000100:00000001:3.0:1787763865.661839:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763865.661840:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497721344 : -115097211830272 : ffff9751d7930000)
00000100:00000040:3.0:1787763865.661845:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930000 x1874599832667648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763890 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.661850:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763865.661852:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763865.661853:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763865.661855:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832667648, offset 0
00000400:00000200:3.0:1787763865.661858:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763865.661861:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763865.661863:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763865.661866:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763865.661867:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763865.661869:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763865.661871:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763865.661873:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58451:58451:256:4294967295] 192.168.203.110@tcp LPNI seq info [58451:58451:8:4294967295]
00000400:00000200:3.0:1787763865.661876:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763865.661880:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763865.661884:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763865.661886:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763865.661889:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763865.661891:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787763865.661894:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763865.661896:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787763865.661899:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763865.661900:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763865.661903:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763865.661905:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763865.661908:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763865.661909:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763865.661912:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763865.661914:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930000 x1874599832667648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763890 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.661919:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763865.661922:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930000 x1874599832667648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763890 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.661926:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763865.661928:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763865.661930:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763865.661933:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787763865.661936:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763865.661938:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316252704
00000100:00000001:3.0:1787763865.661941:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316253940
00000100:00000001:3.0:1787763865.661944:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316252704
00000100:00000001:3.0:1787763865.661946:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316253940
00000100:00000001:3.0:1787763865.661949:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763865.661951:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763865.661953:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763865.661957:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763865/real 1787763865]  req@ffff9751d7930000 x1874599832667648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763865.661965:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763865.661967:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763865.661969:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763865.661971:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763865.661974:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763865.661976:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763865.661980:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930000 x1874599832667648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763865.661989:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930000 x1874599832667648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662004:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763865.662005:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763865.662007:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763865.662008:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763865.662010:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930000 x1874599832667648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662015:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763865.662018:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930000 x1874599832667648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662022:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763865.662024:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763865.662025:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763865.662026:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763865.662028:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763865.662038:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763865.662039:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763865.662041:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763865.662043:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763865.662046:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930000 x1874599832667648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763865.662051:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832667648:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763865.662054:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763865.662056:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930b40 x1874599832667776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662059:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763865.662060:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763865.662063:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763865.662064:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763865.662066:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763865.662068:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832667776:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763865.662070:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763865.662071:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763865.662072:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763865.662074:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763865.662076:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610400.
02000000:00000001:3.0:1787763865.662078:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763865.662080:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763865.662082:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763865.662085:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763865.662087:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832667776, portal 10
00000100:00000001:3.0:1787763865.662089:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763865.662090:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497724224 : -115097211827392 : ffff9751d7930b40)
00000100:00000040:3.0:1787763865.662094:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930b40 x1874599832667776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763890 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662099:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763865.662101:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763865.662102:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763865.662104:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832667776, offset 0
00000400:00000200:3.0:1787763865.662107:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763865.662109:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763865.662112:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763865.662114:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763865.662116:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763865.662117:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763865.662119:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763865.662121:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58452:58452:256:4294967295] 192.168.203.110@tcp LPNI seq info [58452:58452:8:4294967295]
00000400:00000200:3.0:1787763865.662124:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763865.662129:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763865.662184:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763865.662189:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763865.662194:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763865.662197:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787763865.662203:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763865.662206:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787763865.662212:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763865.662215:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763865.662219:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763865.662225:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763865.662228:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763865.662230:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763865.662232:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763865.662235:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930b40 x1874599832667776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763890 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662243:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763865.662247:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930b40 x1874599832667776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763890 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662254:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763865.662256:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763865.662260:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763865.662263:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787763865.662267:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763865.662269:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316252693
00000100:00000001:3.0:1787763865.662272:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316253940
00000100:00000001:3.0:1787763865.662275:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316252693
00000100:00000001:3.0:1787763865.662277:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316253940
00000100:00000001:3.0:1787763865.662279:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763865.662282:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763865.662284:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763865.662288:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763865/real 1787763865]  req@ffff9751d7930b40 x1874599832667776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763865.662295:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763865.662296:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763865.662298:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763865.662300:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763865.662301:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763865.662303:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763865.662305:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930b40 x1874599832667776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763865.662311:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930b40 x1874599832667776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662315:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763865.662315:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763865.662316:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763865.662317:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763865.662319:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930b40 x1874599832667776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662324:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763865.662327:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930b40 x1874599832667776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662332:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763865.662333:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763865.662334:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763865.662335:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763865.662336:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763865.662337:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763865.662339:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763865.662341:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763865.662342:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763865.662345:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930b40 x1874599832667776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763865.662350:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832667776:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763865.662353:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763865.662355:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7931680 x1874599832667904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662359:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763865.662360:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763865.662361:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763865.662363:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763865.662364:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763865.662366:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832667904:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763865.662368:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763865.662369:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763865.662370:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763865.662371:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763865.662374:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613c00.
02000000:00000001:3.0:1787763865.662375:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763865.662378:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763865.662379:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763865.662381:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763865.662384:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832667904, portal 4
00000100:00000001:3.0:1787763865.662385:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763865.662386:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497727104 : -115097211824512 : ffff9751d7931680)
00000100:00000040:3.0:1787763865.662390:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7931680 x1874599832667904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763890 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662395:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763865.662397:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763865.662399:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763865.662402:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832667904, offset 0
00000400:00000200:3.0:1787763865.662407:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763865.662411:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763865.662415:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763865.662418:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763865.662421:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763865.662424:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763865.662426:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763865.662429:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58453:58453:256:4294967295] 192.168.203.110@tcp LPNI seq info [58453:58453:8:4294967295]
00000400:00000200:3.0:1787763865.662434:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763865.662441:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763865.662446:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763865.662450:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763865.662454:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763865.662456:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787763865.662459:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763865.662462:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787763865.662465:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763865.662466:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763865.662468:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763865.662471:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763865.662474:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763865.662475:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763865.662477:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763865.662479:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7931680 x1874599832667904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763890 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662485:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763865.662487:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7931680 x1874599832667904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763890 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662492:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763865.662493:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763865.662495:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763865.662499:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787763865.662503:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763865.662505:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316252710
00000100:00000001:3.0:1787763865.662508:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316253940
00000100:00000001:3.0:1787763865.662511:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316252710
00000100:00000001:3.0:1787763865.662513:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316253940
00000100:00000001:3.0:1787763865.662515:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763865.662517:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763865.662520:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763865.662524:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763865/real 1787763865]  req@ffff9751d7931680 x1874599832667904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763865.662533:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763865.662535:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763865.662537:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763865.662540:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763865.662542:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763865.662544:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763865.662547:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7931680 x1874599832667904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763865.662553:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7931680 x1874599832667904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662557:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763865.662558:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763865.662560:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763865.662561:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763865.662563:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7931680 x1874599832667904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662568:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763865.662570:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7931680 x1874599832667904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662574:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763865.662577:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763865.662578:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763865.662579:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763865.662580:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763865.662581:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763865.662583:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763865.662585:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763865.662586:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763865.662589:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7931680 x1874599832667904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763865.662593:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832667904:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763865.662596:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763865.662598:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763865.662601:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933480 x1874599832667520/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662606:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763865.662609:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763865.662611:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611800.
02000000:00000001:3.0:1787763865.662613:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763865.662615:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763865.662616:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787763865.662618:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763865.662621:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9400.
02000000:00000001:3.0:1787763865.662623:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763865.662625:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763865.662627:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933480.
00000100:00000001:3.0:1787763865.662631:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763865.662631:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763865.662633:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763865.662635:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930000 x1874599832667648/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662643:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763865.662644:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763865.662645:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613800.
02000000:00000001:3.0:1787763865.662647:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763865.662648:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763865.662649:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763865.662650:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763865.662651:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cac00.
02000000:00000001:3.0:1787763865.662652:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763865.662653:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763865.662653:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930000.
00000100:00000001:3.0:1787763865.662655:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763865.662656:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763865.662657:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763865.662658:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930b40 x1874599832667776/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662662:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763865.662663:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763865.662664:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610400.
02000000:00000001:3.0:1787763865.662665:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763865.662666:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763865.662666:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763865.662667:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763865.662668:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:3.0:1787763865.662669:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763865.662670:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763865.662671:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930b40.
00000100:00000001:3.0:1787763865.662672:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763865.662673:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763865.662673:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763865.662675:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7931680 x1874599832667904/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763890 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763865.662679:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763865.662680:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763865.662681:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613c00.
02000000:00000001:3.0:1787763865.662682:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763865.662683:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763865.662684:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787763865.662685:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763865.662686:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb000.
02000000:00000001:3.0:1787763865.662688:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763865.662688:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763865.662689:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7931680.
00000100:00000001:3.0:1787763865.662691:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763865.662692:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763865.662693:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763865.662710:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763865.662711:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763865.662713:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763865.662714:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763866.300238:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763866.300245:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763866.300247:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763866.300249:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763866.300252:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763866.300255:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763866.300259:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763866.300260:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763866.300262:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787763866.300266:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763866.300266:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763866.300270:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787763866.300272:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763866.300273:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763866.300278:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763866.300280:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763866.300281:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763866.300285:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763866.300286:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763866.300290:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763866.300291:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763866.300326:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763866.300331:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763866.300335:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763866.684326:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763866.684335:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763866.684338:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763866.684344:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763866.684350:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763866.684355:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763866.684357:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763866.684360:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763866.684364:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763866.940195:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316253940, at 4316253952
00000100:00000010:3.0:1787763866.940208:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40960.
00000100:00000040:2.0:1787763866.940244:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316253940, at 4316253952
00000100:00000010:2.0:1787763866.940251:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40a80.
00000100:00000001:2.0:1787763867.324212:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763867.324219:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763867.324221:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763867.324226:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763867.324230:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763867.324233:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763867.324236:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763867.324238:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763867.324240:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763867.324257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763867.324264:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763867.324271:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763867.324273:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763867.324277:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763867.324278:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763867.324331:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763867.324336:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763867.324340:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763867.708247:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763867.708254:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763867.708255:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763867.708259:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763867.708263:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763867.708266:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763867.708268:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763867.708271:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763867.708273:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763868.348212:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763868.348213:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763868.348218:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763868.348219:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763868.348223:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763868.348225:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763868.348227:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763868.348228:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763868.348232:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763868.348234:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763868.348236:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763868.348237:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763868.348240:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763868.348242:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763868.348244:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763868.348296:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763868.348301:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763868.348305:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763868.732271:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763868.732279:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763868.732282:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763868.732287:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763868.732290:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763868.732293:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763868.732296:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763868.732298:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763868.732301:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763869.372211:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763869.372212:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763869.372218:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763869.372220:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763869.372224:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763869.372225:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763869.372226:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763869.372233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763869.372234:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763869.372234:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763869.372240:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763869.372244:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763869.372247:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763869.372251:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763869.372255:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763869.372291:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763869.372295:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763869.372299:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763869.756233:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763869.756239:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763869.756241:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763869.756246:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763869.756250:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763869.756253:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763869.756255:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763869.756257:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763869.756259:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763870.396238:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763870.396238:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763870.396245:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763870.396246:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763870.396249:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763870.396254:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763870.396255:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763870.396259:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763870.396260:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763870.396264:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763870.396267:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763870.396267:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763870.396272:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763870.396273:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763870.396278:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763870.396315:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763870.396319:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763870.396321:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763870.780214:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763870.780220:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763870.780222:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763870.780227:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:0.0:1787763870.780230:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787763870.780232:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763870.780236:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787763870.780240:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787763870.780240:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763870.780244:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763870.780246:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1787763870.780248:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763870.780252:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763870.780259:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763870.780266:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21585
00000400:00000200:0.0:1787763870.780273:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763870.780277:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763870.780281:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763870.780283:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763870.780285:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763870.780287:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763870.780289:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763870.780292:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763870.780294:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763870.780297:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763870.780300:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763870.780301:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763870.780303:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763870.780305:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763870.780306:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763870.780308:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763870.780309:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763870.780311:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763870.780313:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763870.780315:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763870.780317:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763870.780319:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763870.780321:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763870.780323:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763870.780325:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763870.780333:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
00000020:00000040:0.0:1787763870.780336:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763870.780339:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763870.780340:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763870.780342:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763870.780345:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763870.780348:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763870.780355:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763870.780368:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763870.780375:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832668032/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763870.780384:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763870.780387:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763870.780391:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832668032/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763870.780410:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763870.780412:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763870.780415:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763870.780416:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763870.780417:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763870.780419:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763870.780422:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21585
00000400:00000200:0.0:1787763870.780425:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763870.780427:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787763870.780428:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:0.0:1787763870.780429:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763870.780434:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763870.780434:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763870.780438:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1787763870.780439:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763870.780443:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763870.780446:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:3.0:1787763870.780447:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f840 x1874599832668032/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787763870.780448:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763870.780451:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763870.780454:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763870.780457:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763870.780460:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:3.0:1787763870.780463:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:0.0:1787763870.780464:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:3.0:1787763870.780465:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763870.780469:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:3.0:1787763870.780470:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1787763870.780473:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1787763870.780474:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1787763870.780478:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:3.0:1787763870.780479:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763870.780481:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763870.780486:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00100000:3.0:1787763870.780486:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832668032:192.168.203.110@tcp:250:kworker.0
00000100:00080000:0.0:1787763870.780492:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763870.780493:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1787763870.780495:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763870.780498:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763870.780499:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763870.780501:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763870.780504:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1787763870.780505:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763870.780509:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763870.780509:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613c00.
00000100:00000001:0.0:1787763870.780511:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1787763870.780514:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1787763870.780517:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:3.0:1787763870.780522:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000008:00000001:0.0:1787763870.780526:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787763870.780528:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000100:00000001:0.0:1787763870.780530:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787763870.780536:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000010:0.0:1787763870.780538:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787763870.780543:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000100:00000200:3.0:1787763870.780543:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832668032, portal 25
00000100:00000001:3.0:1787763870.780548:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:0.0:1787763870.780550:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1787763870.780550:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049532480 : -115096660019136 : ffff9751f876f840)
02000000:00000001:0.0:1787763870.780554:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763870.780558:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f840 x1874599832668032/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763895 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763870.780560:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763870.780563:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763870.780568:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763870.780568:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763870.780573:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
02000000:00000010:0.0:1787763870.780577:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000400:00000010:3.0:1787763870.780577:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000001:0.0:1787763870.780584:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787763870.780584:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832668032, offset 0
00000100:00100000:0.0:1787763870.780592:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832668160/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787763870.780593:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763870.780601:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1787763870.780602:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763870.780605:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763870.780611:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832668160/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787763870.780613:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763870.780621:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1787763870.780624:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763870.780627:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1787763870.780627:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00080000:0.0:1787763870.780632:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:3.0:1787763870.780633:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1787763870.780634:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763870.780637:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1787763870.780637:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:0.0:1787763870.780642:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1787763870.780645:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58454:58454:256:4294967295] 192.168.203.110@tcp LPNI seq info [58454:58454:8:4294967295]
00000100:00080000:0.0:1787763870.780649:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21585
00000400:00000200:3.0:1787763870.780653:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763870.780657:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763870.780666:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1787763870.780666:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00080000:0.0:1787763870.780673:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000200:3.0:1787763870.780677:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787763870.780682:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000010:3.0:1787763870.780683:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000100:00000001:0.0:1787763870.780689:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1787763870.780693:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763870.780697:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000200:3.0:1787763870.780699:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000100:00000001:0.0:1787763870.780703:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763870.780709:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763870.780710:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763870.780711:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763870.780713:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000200:3.1:1787763870.780714:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787763870.780716:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:3.1:1787763870.780719:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:0.0:1787763870.780721:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000100:3.0:1787763870.780727:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763870.780730:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000100:00000040:0.0:1787763870.780731:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763870.780736:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787763870.780736:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:0.0:1787763870.780741:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:3.0:1787763870.780745:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1787763870.780748:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:3.0:1787763870.780750:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000020:00000040:0.0:1787763870.780754:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000010:3.0:1787763870.780754:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763870.780760:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:0.0:1787763870.780762:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000200:3.0:1787763870.780766:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f840 x1874599832668032/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763895 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763870.780768:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763870.780772:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763870.780777:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763870.780779:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:0.0:1787763870.780780:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763870.780785:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:3.0:1787763870.780785:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f840 x1874599832668032/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763895 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1787763870.780789:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763870.780794:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763870.780799:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763870.780801:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763870.780804:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000100:00000001:3.0:1787763870.780804:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000040:0.0:1787763870.780809:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763870.780811:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1787763870.780811:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763870.780814:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763870.780818:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1787763870.780818:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787763870.780820:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1787763870.780822:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316259059
02000000:00000001:0.0:1787763870.780824:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763870.780831:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000010:3.0:1787763870.780831:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40960.
00000100:00000001:0.0:1787763870.780838:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763870.780840:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00100000:0.0:1787763870.780844:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832668288/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787763870.780848:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316259059
00000100:00000001:0.0:1787763870.780853:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000010:3.0:1787763870.780856:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d404e0.
00000100:00000001:0.0:1787763870.780857:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763870.780864:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832668288/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.780864:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787763870.780871:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763870.780873:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763870.780876:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763870.780877:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763870.780883:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1787763870.780884:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00080000:0.0:1787763870.780885:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763870.780889:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763870.780891:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:3.0:1787763870.780893:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763870/real 1787763870]  req@ffff9751f876f840 x1874599832668032/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787763870.780894:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763870.780898:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763870.780902:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21585
00000400:00000200:0.0:1787763870.780910:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763870.780917:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000001:3.0:1787763870.780917:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763870.780921:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00080000:0.0:1787763870.780922:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000010:3.0:1787763870.780924:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000100:00000040:0.0:1787763870.780927:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:3.0:1787763870.780929:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000100:00000001:0.0:1787763870.780930:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:3.0:1787763870.780933:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000040:0.0:1787763870.780935:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763870.780937:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:0.0:1787763870.780938:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763870.780942:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763870.780944:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000100:00000200:3.0:1787763870.780945:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f840 x1874599832668032/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787763870.780946:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763870.780949:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763870.780954:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763870.780960:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000200:3.0:1787763870.780963:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f840 x1874599832668032/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763870.780967:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763870.780971:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763870.780976:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763870.780979:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787763870.780979:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763870.780982:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:0.0:1787763870.780983:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:3.0:1787763870.780985:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00080000:0.0:1787763870.780989:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763870.780989:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763870.780994:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763870.780998:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:3.0:1787763870.780998:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f840 x1874599832668032/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763870.781000:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763870.781005:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763870.781008:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763870.781012:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763870.781016:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763870.781017:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763870.781018:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763870.781025:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000100:00000040:3.0:1787763870.781028:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f840 x1874599832668032/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787763870.781029:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763870.781034:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763870.781036:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763870.781040:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763870.781043:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763870.781045:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763870.781050:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763870.781055:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763870.781063:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832668416/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787763870.781063:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763870.781069:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763870.781073:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763870.781076:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763870.781077:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763870.781080:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763870.781080:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763870.781084:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:0.0:1787763870.781087:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832668416/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787763870.781089:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763870.781097:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763870.781103:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763870.781103:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787763870.781110:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21595)
00000100:00000040:3.0:1787763870.781110:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f840 x1874599832668032/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763870.781149:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832668032:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787763870.781154:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763870.781156:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763870.781160:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f840 x1874599832668032/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781165:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763870.781168:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763870.781170:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613c00.
02000000:00000001:3.0:1787763870.781174:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763870.781176:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763870.781178:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787763870.781180:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763870.781182:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016800.
02000000:00000001:3.0:1787763870.781185:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763870.781186:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763870.781187:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f840.
00000100:00000001:3.0:1787763870.781189:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763870.781190:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763870.781191:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763870.781197:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763870.781198:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763870.781200:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763870.781201:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763870.781202:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763870.781205:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e940 x1874599832668160/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781209:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763870.781210:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763870.781212:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763870.781213:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763870.781215:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763870.781217:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832668160:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763870.781219:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763870.781221:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763870.781222:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763870.781223:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763870.781225:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613c00.
02000000:00000001:3.0:1787763870.781227:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763870.781230:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763870.781232:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763870.781234:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763870.781236:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832668160, portal 10
00000100:00000001:3.0:1787763870.781238:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763870.781239:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049528640 : -115096660022976 : ffff9751f876e940)
00000100:00000040:3.0:1787763870.781243:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e940 x1874599832668160/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763895 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781247:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763870.781249:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763870.781250:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763870.781252:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832668160, offset 0
00000400:00000200:3.0:1787763870.781255:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763870.781257:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763870.781260:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763870.781263:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763870.781265:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763870.781266:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763870.781268:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763870.781270:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58455:58455:256:4294967295] 192.168.203.110@tcp LPNI seq info [58455:58455:8:4294967295]
00000400:00000200:3.0:1787763870.781273:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763870.781277:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763870.781280:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763870.781283:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763870.781285:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763870.781288:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787763870.781291:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763870.781292:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787763870.781295:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763870.781296:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763870.781298:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763870.781301:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763870.781303:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763870.781304:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763870.781306:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763870.781308:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e940 x1874599832668160/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763895 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781313:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763870.781315:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e940 x1874599832668160/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763895 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781319:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763870.781320:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763870.781322:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763870.781325:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787763870.781327:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763870.781328:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316257824
00000100:00000001:3.0:1787763870.781331:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316259059
00000100:00000001:3.0:1787763870.781332:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316257824
00000100:00000001:3.0:1787763870.781334:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316259059
00000100:00000001:3.0:1787763870.781336:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763870.781337:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763870.781338:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763870.781340:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763870/real 1787763870]  req@ffff9751f876e940 x1874599832668160/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763870.781345:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763870.781346:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763870.781347:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763870.781348:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763870.781349:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763870.781351:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763870.781353:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e940 x1874599832668160/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763870.781358:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e940 x1874599832668160/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781362:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763870.781363:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763870.781364:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763870.781365:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763870.781367:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e940 x1874599832668160/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781371:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763870.781373:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e940 x1874599832668160/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781378:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763870.781379:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763870.781381:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763870.781382:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763870.781383:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763870.781393:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763870.781394:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763870.781397:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763870.781398:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763870.781401:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e940 x1874599832668160/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763870.781406:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832668160:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763870.781408:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763870.781410:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cf00 x1874599832668288/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781414:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763870.781415:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763870.781416:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763870.781417:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763870.781418:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763870.781420:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832668288:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763870.781422:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763870.781423:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763870.781425:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763870.781426:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763870.781428:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610400.
02000000:00000001:3.0:1787763870.781429:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763870.781431:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763870.781432:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763870.781434:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763870.781435:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832668288, portal 10
00000100:00000001:3.0:1787763870.781437:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763870.781438:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049521920 : -115096660029696 : ffff9751f876cf00)
00000100:00000040:3.0:1787763870.781442:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cf00 x1874599832668288/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763895 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781446:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763870.781448:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763870.781449:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763870.781450:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832668288, offset 0
00000400:00000200:3.0:1787763870.781453:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763870.781455:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763870.781456:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763870.781458:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763870.781460:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763870.781461:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763870.781462:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763870.781465:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58456:58456:256:4294967295] 192.168.203.110@tcp LPNI seq info [58456:58456:8:4294967295]
00000400:00000200:3.0:1787763870.781467:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763870.781471:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763870.781474:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763870.781476:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763870.781478:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763870.781480:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787763870.781482:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763870.781484:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787763870.781486:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763870.781488:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763870.781489:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763870.781491:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763870.781493:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763870.781494:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763870.781496:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763870.781498:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cf00 x1874599832668288/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763895 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781503:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763870.781505:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cf00 x1874599832668288/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763895 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781509:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763870.781510:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763870.781512:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763870.781514:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787763870.781515:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763870.781516:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316257812
00000100:00000001:3.0:1787763870.781518:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316259059
00000100:00000001:3.0:1787763870.781519:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316257812
00000100:00000001:3.0:1787763870.781521:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316259059
00000100:00000001:3.0:1787763870.781522:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763870.781523:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763870.781524:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763870.781527:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763870/real 1787763870]  req@ffff9751f876cf00 x1874599832668288/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763870.781531:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763870.781532:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763870.781533:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763870.781534:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763870.781535:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763870.781536:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763870.781538:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cf00 x1874599832668288/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763870.781544:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cf00 x1874599832668288/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781547:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763870.781548:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763870.781549:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763870.781550:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763870.781553:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cf00 x1874599832668288/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781558:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763870.781560:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cf00 x1874599832668288/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781564:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763870.781565:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763870.781566:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763870.781567:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763870.781568:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763870.781569:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763870.781571:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763870.781572:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763870.781574:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763870.781576:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cf00 x1874599832668288/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763870.781580:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832668288:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763870.781583:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763870.781585:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832668416/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781589:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763870.781589:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763870.781591:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763870.781592:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763870.781593:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763870.781595:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832668416:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763870.781597:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763870.781599:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763870.781600:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763870.781601:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763870.781603:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613800.
02000000:00000001:3.0:1787763870.781604:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763870.781606:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763870.781608:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763870.781610:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763870.781611:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832668416, portal 4
00000100:00000001:3.0:1787763870.781613:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763870.781614:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
00000100:00000040:3.0:1787763870.781617:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832668416/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763895 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781621:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763870.781623:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763870.781624:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763870.781625:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832668416, offset 0
00000400:00000200:3.0:1787763870.781628:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763870.781629:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763870.781631:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763870.781633:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763870.781634:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763870.781636:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763870.781637:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763870.781639:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58457:58457:256:4294967295] 192.168.203.110@tcp LPNI seq info [58457:58457:8:4294967295]
00000400:00000200:3.0:1787763870.781642:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763870.781646:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763870.781649:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763870.781650:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763870.781652:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763870.781654:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787763870.781656:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763870.781658:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787763870.781661:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763870.781662:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763870.781664:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763870.781666:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763870.781668:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763870.781669:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763870.781670:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763870.781672:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876ed00 x1874599832668416/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763895 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781677:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763870.781679:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832668416/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763895 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781684:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763870.781685:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763870.781686:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763870.781688:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787763870.781690:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763870.781691:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316257829
00000100:00000001:3.0:1787763870.781693:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316259059
00000100:00000001:3.0:1787763870.781694:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316257829
00000100:00000001:3.0:1787763870.781695:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316259059
00000100:00000001:3.0:1787763870.781697:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763870.781698:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763870.781699:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763870.781701:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763870/real 1787763870]  req@ffff9751f876ed00 x1874599832668416/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763870.781705:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763870.781706:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763870.781707:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763870.781709:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763870.781710:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763870.781711:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763870.781713:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832668416/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763870.781719:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832668416/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781722:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763870.781723:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763870.781724:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763870.781725:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763870.781727:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832668416/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781731:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763870.781733:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832668416/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781737:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763870.781738:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763870.781739:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763870.781740:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763870.781741:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763870.781742:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763870.781743:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763870.781745:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763870.781746:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763870.781749:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832668416/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763870.781753:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832668416:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763870.781755:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763870.781756:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763870.781759:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e940 x1874599832668160/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781763:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763870.781764:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763870.781766:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613c00.
02000000:00000001:3.0:1787763870.781767:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763870.781768:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763870.781769:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763870.781770:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763870.781771:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015c00.
02000000:00000001:3.0:1787763870.781773:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763870.781773:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763870.781774:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e940.
00000100:00000001:3.0:1787763870.781777:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763870.781778:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763870.781779:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763870.781781:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cf00 x1874599832668288/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781785:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763870.781786:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763870.781787:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610400.
02000000:00000001:3.0:1787763870.781788:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763870.781789:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763870.781790:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763870.781791:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763870.781792:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:3.0:1787763870.781793:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763870.781793:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763870.781794:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cf00.
00000100:00000001:3.0:1787763870.781796:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763870.781797:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763870.781798:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763870.781800:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832668416/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763895 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763870.781803:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763870.781805:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763870.781806:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613800.
02000000:00000001:3.0:1787763870.781807:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763870.781808:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763870.781809:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787763870.781810:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763870.781811:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014800.
02000000:00000001:3.0:1787763870.781812:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763870.781813:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763870.781813:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:3.0:1787763870.781815:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763870.781815:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763870.781816:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763870.781818:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763870.781819:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763870.781820:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763870.781821:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763871.420194:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763871.420196:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763871.420202:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763871.420204:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763871.420207:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763871.420210:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763871.420212:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763871.420215:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763871.420215:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763871.420221:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763871.420221:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763871.420226:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763871.420226:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763871.420228:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763871.420230:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787763871.420232:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763871.420234:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763871.420237:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763871.420239:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763871.420240:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763871.420242:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763871.420299:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763871.420303:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763871.420306:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763871.804237:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763871.804248:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763871.804252:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763871.804259:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763871.804264:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763871.804269:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763871.804272:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763871.804275:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763871.804279:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763872.060192:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316259059, at 4316259072
00000100:00000010:3.0:1787763872.060205:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d404e0.
00000100:00000040:2.0:1787763872.060257:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316259059, at 4316259072
00000100:00000010:2.0:1787763872.060267:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40960.
00000100:00000001:2.0:1787763872.444237:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763872.444238:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763872.444243:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763872.444246:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763872.444250:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763872.444250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763872.444252:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763872.444257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763872.444258:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763872.444259:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763872.444265:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763872.444269:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763872.444272:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763872.444275:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763872.444278:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763872.444321:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763872.444326:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763872.444330:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763872.828209:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763872.828215:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763872.828217:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763872.828222:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763872.828225:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763872.828228:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763872.828230:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763872.828233:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763872.828235:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763873.468187:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763873.468190:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763873.468194:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763873.468196:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763873.468198:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787763873.468199:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763873.468202:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763873.468204:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763873.468206:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763873.468209:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763873.468211:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763873.468212:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763873.468215:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763873.468215:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763873.468219:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763873.468220:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763873.468222:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763873.468223:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763873.468229:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763873.468229:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763873.468232:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763873.468269:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763873.468272:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763873.468275:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763873.852225:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763873.852231:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763873.852233:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763873.852238:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763873.852242:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763873.852246:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763873.852249:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763873.852252:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763873.852256:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763874.492216:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763874.492218:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763874.492222:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763874.492223:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763874.492227:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763874.492229:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763874.492232:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763874.492232:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763874.492238:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763874.492241:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763874.492242:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763874.492244:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763874.492245:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763874.492248:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763874.492251:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763874.492301:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763874.492305:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763874.492308:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763874.876190:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763874.876197:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763874.876199:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763874.876203:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763874.876207:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763874.876210:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763874.876212:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763874.876214:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763874.876216:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763875.516184:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763875.516191:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763875.516193:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763875.516198:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763875.516202:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763875.516205:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.516207:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763875.516210:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.516212:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763875.516229:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763875.516233:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763875.516237:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763875.516237:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763875.516240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763875.516241:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763875.516296:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763875.516300:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763875.516303:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763875.900247:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763875.900259:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763875.900261:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763875.900266:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763875.900272:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763875.900283:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21590
00000400:00000200:0.0:1787763875.900294:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763875.900296:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000400:00000200:0.0:1787763875.900303:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787763875.900303:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763875.900309:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763875.900311:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763875.900320:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00100000:2.0:1787763875.900320:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763875.900331:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763875.900332:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763875.900345:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763875.900346:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763875.900357:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763875.900357:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763875.900365:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1787763875.900366:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763875.900371:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000100:00000001:2.0:1787763875.900374:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763875.900375:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763875.900384:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763875.900388:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763875.900390:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763875.900394:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763875.900396:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763875.900398:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763875.900400:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763875.900402:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763875.900405:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763875.900406:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763875.900409:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763875.900413:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763875.900415:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763875.900418:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763875.900420:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763875.900430:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763875.900433:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763875.900437:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763875.900440:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763875.900443:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763875.900445:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763875.900449:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763875.900459:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763875.900468:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763875.900480:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832668544/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763875.900499:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763875.900504:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763875.900510:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832668544/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763875.900540:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763875.900545:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763875.900551:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763875.900554:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763875.900558:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763875.900566:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763875.900566:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763875.900574:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1787763875.900579:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21590
00000100:00000001:2.0:1787763875.900579:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000400:00000200:0.0:1787763875.900592:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:2.0:1787763875.900597:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f0c0 x1874599832668544/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787763875.900602:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763875.900611:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763875.900624:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763875.900625:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763875.900631:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763875.900631:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763875.900635:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763875.900638:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763875.900640:0:304484:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:2.0:1787763875.900641:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:0.0:1787763875.900642:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763875.900645:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763875.900647:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
02000000:00000001:2.0:1787763875.900649:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:0.0:1787763875.900650:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763875.900655:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:2.0:1787763875.900660:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763875.900661:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763875.900667:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763875.900675:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763875.900679:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00100000:2.0:1787763875.900680:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832668544:192.168.203.110@tcp:250:kworker.0
00000020:00000040:0.0:1787763875.900681:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763875.900686:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763875.900691:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763875.900695:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763875.900702:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763875.900703:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763875.900706:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763875.900709:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1787763875.900711:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:0.0:1787763875.900715:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1787763875.900716:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1787763875.900719:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763875.900725:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763875.900725:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:0.0:1787763875.900734:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763875.900737:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000010:2.0:1787763875.900738:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:0.0:1787763875.900741:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763875.900745:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763875.900746:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763875.900747:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763875.900751:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763875.900754:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763875.900759:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000400:00000200:2.0:1787763875.900760:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763875.900763:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763875.900768:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000100:00100000:0.0:1787763875.900770:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832668672/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763875.900780:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:2.0:1787763875.900780:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ece58.
00000100:00000001:0.0:1787763875.900787:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763875.900793:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832668544, portal 25
00000100:00000040:0.0:1787763875.900795:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832668672/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.900802:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763875.900810:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763875.900810:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049530560 : -115096660021056 : ffff9751f876f0c0)
00000100:00080000:0.0:1787763875.900818:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763875.900822:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000040:2.0:1787763875.900824:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f0c0 x1874599832668544/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763900 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763875.900825:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763875.900830:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763875.900839:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763875.900847:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:0.0:1787763875.900853:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21590
00000100:00000040:2.0:1787763875.900861:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763875.900865:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763875.900867:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec990.
00000100:00000200:2.0:1787763875.900882:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832668544, offset 0
00000400:00000200:0.0:1787763875.900883:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763875.900894:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1787763875.900902:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787763875.900903:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763875.900909:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763875.900912:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763875.900914:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763875.900921:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:2.0:1787763875.900922:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:0.0:1787763875.900923:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763875.900927:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763875.900929:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763875.900931:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763875.900934:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:2.0:1787763875.900940:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000040:0.0:1787763875.900941:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763875.900947:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763875.900951:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763875.900953:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763875.900954:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:2.0:1787763875.900956:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:0.0:1787763875.900960:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763875.900963:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763875.900964:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763875.900967:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1787763875.900969:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
02000000:00000001:0.0:1787763875.900970:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763875.900975:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763875.900977:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:2.0:1787763875.900981:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000008:00000001:0.0:1787763875.900982:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763875.900987:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763875.900991:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000400:00000200:2.0:1787763875.900995:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000020:00000040:0.0:1787763875.900996:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763875.901004:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763875.901006:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763875.901011:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763875.901013:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763875.901015:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763875.901017:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58458:58458:256:4294967295] 192.168.203.110@tcp LPNI seq info [58458:58458:8:4294967295]
02000000:00000010:0.0:1787763875.901020:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787763875.901024:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763875.901033:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832668800/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787763875.901035:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763875.901047:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763875.901049:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763875.901056:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832668800/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787763875.901073:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1787763875.901075:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763875.901081:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763875.901102:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763875.901105:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763875.901111:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000800:00000200:2.0:1787763875.901111:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763875.901117:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763875.901119:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763875.901124:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763875.901127:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21590
00000800:00000010:2.0:1787763875.901128:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f400.
00000400:00000200:0.0:1787763875.901160:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763875.901165:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763875.901167:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763875.901171:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763875.901175:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763875.901178:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763875.901186:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:2.0:1787763875.901188:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787763875.901190:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763875.901195:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000800:00000200:2.0:1787763875.901197:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:0.0:1787763875.901198:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763875.901201:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763875.901208:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763875.901217:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000800:00000200:2.1:1787763875.901218:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763875.901228:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000100:00000040:0.0:1787763875.901230:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763875.901236:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763875.901242:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763875.901245:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763875.901252:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000800:00000100:2.0:1787763875.901253:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00080000:0.0:1787763875.901258:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000800:00000010:2.0:1787763875.901259:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
00000100:00000001:0.0:1787763875.901260:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763875.901267:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763875.901271:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1787763875.901273:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:0.0:1787763875.901274:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763875.901281:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763875.901286:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763875.901292:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763875.901296:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763875.901297:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000010:0.0:1787763875.901300:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763875.901303:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000400:00000200:2.0:1787763875.901305:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec990
02000000:00000001:0.0:1787763875.901307:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763875.901309:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763875.901311:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763875.901313:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:2.0:1787763875.901313:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec990.
02000000:00000001:0.0:1787763875.901317:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763875.901322:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787763875.901336:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763875.901344:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00100000:0.0:1787763875.901345:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832668928/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763875.901356:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763875.901359:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763875.901364:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f0c0 x1874599832668544/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763900 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763875.901367:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832668928/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763875.901407:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763875.901418:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21600)
00000100:00000001:2.0:1787763875.901433:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763875.901445:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f0c0 x1874599832668544/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763900 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.901460:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763875.901463:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763875.901474:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763875.901483:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763875.901488:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316264180
00000100:00000010:2.0:1787763875.901497:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00000001:2.0:1787763875.901508:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787763875.901534:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316264180
00000100:00000010:2.0:1787763875.901541:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7000.
00000100:00000001:2.0:1787763875.901545:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787763875.901549:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763875.901554:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763875.901557:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763875.901564:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763875/real 1787763875]  req@ffff9751f876f0c0 x1874599832668544/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763875.901577:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763875.901580:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763875.901584:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787763875.901588:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000400:00000010:2.0:1787763875.901591:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000100:00000001:2.0:1787763875.901595:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763875.901600:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f0c0 x1874599832668544/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763875.901614:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f0c0 x1874599832668544/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.901625:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763875.901628:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.901631:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763875.901634:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763875.901641:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f0c0 x1874599832668544/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.901653:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763875.901660:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f0c0 x1874599832668544/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.901674:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763875.901678:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.901681:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763875.901683:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.901687:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763875.901690:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763875.901695:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763875.901702:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763875.901706:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763875.901713:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f0c0 x1874599832668544/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763875.901726:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832668544:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787763875.901737:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.901741:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763875.901746:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f0c0 x1874599832668544/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.901756:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763875.901760:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763875.901764:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787763875.901769:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763875.901772:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763875.901775:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787763875.901778:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763875.901782:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017000.
02000000:00000001:2.0:1787763875.901788:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763875.901791:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763875.901793:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f0c0.
00000100:00000001:2.0:1787763875.901798:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763875.901800:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.901804:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.901813:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763875.901815:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763875.901819:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763875.901821:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763875.901824:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763875.901831:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832668672/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.901841:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763875.901843:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763875.901846:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763875.901850:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763875.901854:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763875.901858:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832668672:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763875.901863:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763875.901867:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763875.901870:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763875.901873:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763875.901878:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787763875.901881:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763875.901903:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763875.901908:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787763875.901915:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ece58.
00000100:00000200:2.0:1787763875.901920:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832668672, portal 10
00000100:00000001:2.0:1787763875.901923:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763875.901925:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00000040:2.0:1787763875.901938:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832668672/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763900 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.901950:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763875.901954:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763875.901959:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec990.
00000100:00000200:2.0:1787763875.901963:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832668672, offset 0
00000400:00000200:2.0:1787763875.901971:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763875.901978:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763875.901985:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763875.901991:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763875.901996:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763875.902001:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763875.902005:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763875.902011:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58459:58459:256:4294967295] 192.168.203.110@tcp LPNI seq info [58459:58459:8:4294967295]
00000400:00000200:2.0:1787763875.902018:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763875.902030:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763875.902039:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763875.902045:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f400.
00000800:00000200:2.0:1787763875.902053:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763875.902058:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787763875.902068:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763875.902073:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1787763875.902081:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763875.902104:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
00000400:00000200:2.0:1787763875.902128:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763875.902134:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763875.902139:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec990
00000400:00000010:2.0:1787763875.902141:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec990.
00000100:00000001:2.0:1787763875.902145:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763875.902180:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e580 x1874599832668672/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763900 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.902190:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763875.902193:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832668672/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763900 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.902200:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763875.902202:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763875.902206:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763875.902211:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787763875.902214:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763875.902216:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316262945
00000100:00000001:2.0:1787763875.902220:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316264180
00000100:00000001:2.0:1787763875.902224:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316262945
00000100:00000001:2.0:1787763875.902227:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316264180
00000100:00000001:2.0:1787763875.902231:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763875.902234:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763875.902237:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763875.902243:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763875/real 1787763875]  req@ffff9751f876e580 x1874599832668672/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763875.902255:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763875.902257:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763875.902260:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787763875.902264:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000400:00000010:2.0:1787763875.902267:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000100:00000001:2.0:1787763875.902271:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763875.902276:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832668672/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763875.902303:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832668672/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.902311:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763875.902312:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.902314:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763875.902315:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763875.902320:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832668672/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.902327:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763875.902332:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832668672/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.902340:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763875.902343:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.902345:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763875.902346:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.902349:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763875.902362:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763875.902364:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763875.902368:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763875.902371:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763875.902375:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832668672/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763875.902384:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832668672:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763875.902389:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763875.902393:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832668800/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.902406:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763875.902408:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763875.902410:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763875.902413:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763875.902415:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763875.902417:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832668800:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763875.902421:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763875.902423:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763875.902425:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763875.902427:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763875.902430:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763875.902432:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763875.902436:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763875.902438:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787763875.902442:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ece58.
00000100:00000200:2.0:1787763875.902445:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832668800, portal 10
00000100:00000001:2.0:1787763875.902447:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763875.902448:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000100:00000040:2.0:1787763875.902455:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832668800/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763900 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.902461:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763875.902464:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763875.902467:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec990.
00000100:00000200:2.0:1787763875.902470:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832668800, offset 0
00000400:00000200:2.0:1787763875.902475:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763875.902479:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763875.902483:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763875.902486:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763875.902489:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763875.902492:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763875.902495:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763875.902499:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58460:58460:256:4294967295] 192.168.203.110@tcp LPNI seq info [58460:58460:8:4294967295]
00000400:00000200:2.0:1787763875.902503:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763875.902511:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763875.902515:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763875.902519:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f400.
00000800:00000200:2.0:1787763875.902524:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763875.902527:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787763875.902531:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763875.902534:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1787763875.902539:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763875.902541:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
00000400:00000200:2.0:1787763875.902544:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763875.902548:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763875.902568:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec990
00000400:00000010:2.0:1787763875.902571:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec990.
00000100:00000001:2.0:1787763875.902574:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763875.902580:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e1c0 x1874599832668800/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763900 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.902594:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763875.902599:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832668800/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763900 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.902610:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763875.902613:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763875.902617:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763875.902622:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787763875.902627:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763875.902629:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316262933
00000100:00000001:2.0:1787763875.902633:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316264180
00000100:00000001:2.0:1787763875.902636:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316262933
00000100:00000001:2.0:1787763875.902640:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316264180
00000100:00000001:2.0:1787763875.902643:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763875.902646:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763875.902649:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763875.902655:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763875/real 1787763875]  req@ffff9751f876e1c0 x1874599832668800/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763875.902668:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763875.902670:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763875.902673:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787763875.902676:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000400:00000010:2.0:1787763875.902679:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000100:00000001:2.0:1787763875.902682:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763875.902687:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832668800/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763875.902699:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832668800/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.902709:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763875.902711:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.902714:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763875.902716:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763875.902721:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832668800/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.902732:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763875.902739:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832668800/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.902770:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763875.902775:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.902779:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763875.902782:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.902786:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763875.902789:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763875.902794:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763875.902802:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763875.902806:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763875.902829:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832668800/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763875.902840:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832668800:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763875.902847:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763875.902853:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832668928/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.902863:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763875.902865:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763875.902869:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763875.902871:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763875.902874:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763875.902879:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832668928:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763875.902884:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763875.902888:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763875.902890:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763875.902893:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763875.902898:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763875.902917:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763875.902920:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763875.902923:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787763875.902926:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ece58.
00000100:00000200:2.0:1787763875.902929:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832668928, portal 4
00000100:00000001:2.0:1787763875.902931:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763875.902932:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
00000100:00000040:2.0:1787763875.902938:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832668928/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763900 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.902945:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763875.902947:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763875.902950:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec990.
00000100:00000200:2.0:1787763875.902952:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832668928, offset 0
00000400:00000200:2.0:1787763875.902956:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763875.902961:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763875.902964:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763875.902967:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763875.902970:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763875.902973:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763875.902975:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763875.902978:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58461:58461:256:4294967295] 192.168.203.110@tcp LPNI seq info [58461:58461:8:4294967295]
00000400:00000200:2.0:1787763875.902982:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763875.903009:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763875.903019:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763875.903026:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f400.
00000800:00000200:2.0:1787763875.903034:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763875.903039:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787763875.903047:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763875.903069:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1787763875.903076:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763875.903080:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
00000400:00000200:2.0:1787763875.903115:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763875.903137:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763875.903140:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec990
00000400:00000010:2.0:1787763875.903142:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec990.
00000100:00000001:2.0:1787763875.903145:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763875.903149:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876c000 x1874599832668928/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763900 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.903174:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763875.903178:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832668928/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763900 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.903202:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763875.903205:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763875.903211:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763875.903216:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787763875.903220:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763875.903224:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316262951
00000100:00000001:2.0:1787763875.903228:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316264180
00000100:00000001:2.0:1787763875.903232:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316262951
00000100:00000001:2.0:1787763875.903235:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316264180
00000100:00000001:2.0:1787763875.903238:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763875.903242:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763875.903245:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763875.903251:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763875/real 1787763875]  req@ffff9751f876c000 x1874599832668928/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763875.903263:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763875.903265:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763875.903267:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787763875.903271:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000400:00000010:2.0:1787763875.903290:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000100:00000001:2.0:1787763875.903292:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763875.903296:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832668928/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763875.903305:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832668928/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.903312:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763875.903314:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.903316:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763875.903317:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763875.903321:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832668928/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.903328:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763875.903332:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832668928/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.903339:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763875.903342:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.903343:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763875.903345:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.903347:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763875.903348:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763875.903351:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763875.903355:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763875.903358:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763875.903362:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832668928/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763875.903371:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832668928:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763875.903375:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.903376:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763875.903380:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832668672/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.903387:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763875.903389:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763875.903391:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787763875.903394:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763875.903395:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763875.903397:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763875.903400:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763875.903402:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014400.
02000000:00000001:2.0:1787763875.903405:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763875.903406:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763875.903408:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:2.0:1787763875.903412:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763875.903413:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.903415:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763875.903419:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832668800/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.903426:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763875.903427:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763875.903429:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763875.903432:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763875.903433:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763875.903435:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763875.903437:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763875.903439:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:2.0:1787763875.903441:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763875.903442:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763875.903444:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:2.0:1787763875.903446:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763875.903448:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.903449:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763875.903453:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832668928/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763900 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763875.903460:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763875.903462:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763875.903464:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763875.903466:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763875.903468:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763875.903469:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787763875.903471:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763875.903473:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
02000000:00000001:2.0:1787763875.903475:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763875.903477:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763875.903478:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:2.0:1787763875.903481:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763875.903482:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.903484:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763875.903488:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763875.903489:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763875.903492:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763875.903494:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763876.540226:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763876.540233:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763876.540235:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763876.540239:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763876.540243:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763876.540246:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763876.540250:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763876.540250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763876.540259:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763876.540259:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763876.540269:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763876.540271:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763876.540274:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763876.540279:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763876.540284:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763876.540287:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763876.540289:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763876.540291:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763876.540293:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763876.540298:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763876.540300:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763876.540354:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763876.540357:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763876.540360:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763876.924252:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763876.924260:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763876.924263:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763876.924270:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763876.924274:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763876.924278:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763876.924281:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763876.924283:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763876.924287:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763877.180239:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316264180, at 4316264192
00000100:00000010:3.0:1787763877.180257:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7000.
00000100:00000040:2.0:1787763877.180261:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316264180, at 4316264192
00000100:00000010:2.0:1787763877.180278:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00010000:00000001:3.0:1787763877.564283:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763877.564284:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763877.564293:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763877.564295:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763877.564303:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763877.564307:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763877.564318:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763877.564319:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763877.564335:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763877.564336:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763877.564340:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763877.564342:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763877.564354:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763877.564358:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763877.564362:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763877.564445:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763877.564450:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763877.564453:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763877.948269:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763877.948279:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763877.948282:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763877.948288:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763877.948293:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763877.948298:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763877.948302:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763877.948306:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763877.948310:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763878.588236:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763878.588238:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763878.588244:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763878.588246:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763878.588250:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763878.588254:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763878.588256:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763878.588258:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763878.588263:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763878.588265:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763878.588269:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763878.588270:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763878.588278:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763878.588282:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763878.588285:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763878.588357:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763878.588363:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763878.588367:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763878.972285:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763878.972296:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763878.972299:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763878.972306:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763878.972310:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763878.972314:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763878.972318:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763878.972321:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763878.972325:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763879.612239:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763879.612243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763879.612248:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763879.612265:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763879.612267:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763879.612268:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763879.612271:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763879.612275:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763879.612278:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763879.612281:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763879.612286:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763879.612290:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763879.612293:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763879.612297:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763879.612300:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763879.612324:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763879.612327:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763879.612331:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763879.996292:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763879.996299:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763879.996302:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763879.996307:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763879.996311:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763879.996315:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763879.996317:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763879.996320:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763879.996323:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763880.636228:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763880.636232:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763880.636236:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763880.636242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763880.636245:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763880.636245:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763880.636248:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763880.636251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763880.636254:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763880.636255:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763880.636262:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763880.636265:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763880.636268:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763880.636270:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763880.636274:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763880.636320:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763880.636324:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763880.636326:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763881.020199:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763881.020205:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763881.020207:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763881.020212:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763881.020216:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763881.020219:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.020222:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763881.020224:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.020228:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763881.020253:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763881.020259:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763881.020261:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763881.020264:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763881.020269:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763881.020275:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21595
00000400:00000200:0.0:1787763881.020281:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763881.020288:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763881.020291:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763881.020294:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763881.020297:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763881.020299:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763881.020301:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763881.020323:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763881.020325:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763881.020327:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763881.020329:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763881.020331:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763881.020333:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763881.020335:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763881.020337:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763881.020338:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763881.020339:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763881.020341:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763881.020344:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763881.020345:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763881.020347:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763881.020350:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763881.020352:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763881.020354:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763881.020356:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763881.020363:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000020:00000040:0.0:1787763881.020366:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763881.020385:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763881.020386:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763881.020389:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763881.020391:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763881.020394:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763881.020400:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787763881.020406:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763881.020413:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832669056/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763881.020420:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763881.020424:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763881.020427:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832669056/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763881.020442:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763881.020445:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763881.020447:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763881.020448:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763881.020450:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763881.020452:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763881.020454:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21595
00000400:00000200:0.0:1787763881.020456:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763881.020458:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763881.020460:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763881.020463:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763881.020465:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763881.020467:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763881.020468:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763881.020471:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1787763881.020471:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:0.0:1787763881.020475:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763881.020477:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1787763881.020477:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763881.020480:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1787763881.020481:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763881.020485:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763881.020487:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:2.0:1787763881.020490:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876de00 x1874599832669056/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763881.020491:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763881.020496:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763881.020499:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763881.020504:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763881.020508:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00000001:2.0:1787763881.020511:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:0.0:1787763881.020512:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763881.020513:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763881.020517:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763881.020519:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787763881.020520:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763881.020524:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763881.020526:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763881.020530:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832669056:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787763881.020533:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763881.020534:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763881.020538:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763881.020538:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1787763881.020542:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1787763881.020542:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763881.020546:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:0.0:1787763881.020550:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000010:2.0:1787763881.020553:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
00000008:00000001:0.0:1787763881.020554:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763881.020558:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763881.020559:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763881.020566:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
00000400:00000200:2.0:1787763881.020567:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:0.0:1787763881.020569:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763881.020573:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:2.0:1787763881.020573:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
02000000:00000001:0.0:1787763881.020576:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763881.020579:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763881.020580:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763881.020581:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763881.020583:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ece58.
02000000:00000010:0.0:1787763881.020585:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763881.020590:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763881.020591:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832669056, portal 25
00000100:00100000:0.0:1787763881.020595:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832669184/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763881.020597:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763881.020601:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049525760 : -115096660025856 : ffff9751f876de00)
00000100:00000001:0.0:1787763881.020608:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763881.020610:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763881.020615:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876de00 x1874599832669056/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763906 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763881.020616:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832669184/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.020648:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1787763881.020653:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763881.020654:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00080000:0.0:1787763881.020656:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:2.0:1787763881.020659:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec990.
00000100:00080000:0.0:1787763881.020660:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763881.020663:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763881.020666:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:2.0:1787763881.020666:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832669056, offset 0
00000100:00000001:0.0:1787763881.020670:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763881.020675:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21595
00000400:00000200:2.0:1787763881.020679:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763881.020683:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763881.020687:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763881.020688:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763881.020692:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:2.0:1787763881.020693:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763881.020696:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763881.020700:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763881.020702:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787763881.020703:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000020:00000001:0.0:1787763881.020704:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763881.020707:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763881.020710:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:2.0:1787763881.020711:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:0.0:1787763881.020713:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763881.020716:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:2.0:1787763881.020718:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000020:00000001:0.0:1787763881.020719:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:2.0:1787763881.020724:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000040:0.0:1787763881.020725:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763881.020728:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1787763881.020730:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000040:0.0:1787763881.020733:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763881.020738:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787763881.020739:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58462:58462:256:4294967295] 192.168.203.110@tcp LPNI seq info [58462:58462:8:4294967295]
00000020:00000040:0.0:1787763881.020742:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:2.0:1787763881.020746:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:0.0:1787763881.020748:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763881.020752:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763881.020756:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763881.020760:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1787763881.020761:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
02000000:00000001:0.0:1787763881.020765:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763881.020768:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763881.020772:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000800:00000200:2.0:1787763881.020772:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000008:00000001:0.0:1787763881.020778:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1787763881.020780:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f400.
00000100:00000001:0.0:1787763881.020781:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763881.020788:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000020:00000040:0.0:1787763881.020791:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763881.020793:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763881.020793:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763881.020795:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763881.020796:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763881.020797:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787763881.020799:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000010:0.0:1787763881.020801:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000800:00000200:2.0:1787763881.020804:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000100:00000001:0.0:1787763881.020806:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763881.020813:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832669312/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.1:1787763881.020830:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763881.020837:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000100:00000001:0.0:1787763881.020841:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763881.020846:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:2.0:1787763881.020846:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763881.020851:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
00000100:00000040:0.0:1787763881.020854:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832669312/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787763881.020857:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1787763881.020863:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763881.020866:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:0.0:1787763881.020868:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787763881.020871:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec990
00000100:00000040:0.0:1787763881.020872:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000010:2.0:1787763881.020875:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec990.
00000100:00080000:0.0:1787763881.020876:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763881.020896:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763881.020897:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787763881.020897:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:0.0:1787763881.020901:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:2.0:1787763881.020904:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876de00 x1874599832669056/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763906 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763881.020905:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763881.020911:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21595
00000400:00000200:0.0:1787763881.020919:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763881.020924:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787763881.020925:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00080000:0.0:1787763881.020926:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763881.020930:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876de00 x1874599832669056/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763906 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763881.020932:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763881.020936:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763881.020940:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763881.020945:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763881.020949:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1787763881.020951:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787763881.020952:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000100:00000001:2.0:1787763881.020954:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:0.0:1787763881.020955:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763881.020959:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763881.020961:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:2.0:1787763881.020962:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763881.020963:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763881.020971:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:2.0:1787763881.020972:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763881.020977:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316269299
00000100:00000001:0.0:1787763881.020978:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763881.020985:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000010:2.0:1787763881.020985:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00000001:0.0:1787763881.020991:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763881.020995:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000020:00000040:0.0:1787763881.020996:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763881.021002:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763881.021005:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763881.021006:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316269299
02000000:00000001:0.0:1787763881.021008:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000010:2.0:1787763881.021014:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7f00.
02000000:00000001:0.0:1787763881.021015:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763881.021021:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763881.021022:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787763881.021025:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787763881.021028:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000008:00100000:0.0:1787763881.021029:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763881.021034:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763881.021035:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763881.021036:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763881.021042:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000010:0.0:1787763881.021044:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763881.021048:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00100000:2.0:1787763881.021064:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763881/real 1787763881]  req@ffff9751f876de00 x1874599832669056/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763881.021065:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763881.021068:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763881.021073:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763881.021075:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763881.021078:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763881.021086:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000400:00000001:2.0:1787763881.021086:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763881.021089:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:0.0:1787763881.021091:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763881.021092:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787763881.021098:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000100:00100000:0.0:1787763881.021100:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832669440/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:2.0:1787763881.021101:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000100:00000001:2.0:1787763881.021105:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763881.021112:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876de00 x1874599832669056/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763881.021113:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763881.021118:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763881.021126:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832669440/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763881.021130:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876de00 x1874599832669056/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763881.021154:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763881.021158:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21605)
00000100:00000001:2.0:1787763881.021170:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763881.021173:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.021175:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763881.021177:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763881.021181:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876de00 x1874599832669056/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.021186:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763881.021189:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876de00 x1874599832669056/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.021194:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763881.021196:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.021197:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763881.021198:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.021199:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763881.021201:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763881.021203:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763881.021205:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763881.021207:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763881.021210:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876de00 x1874599832669056/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763881.021215:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832669056:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787763881.021219:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.021221:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763881.021223:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876de00 x1874599832669056/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.021229:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763881.021232:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763881.021236:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763881.021239:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763881.021241:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763881.021243:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787763881.021245:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763881.021248:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016c00.
02000000:00000001:2.0:1787763881.021253:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763881.021255:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763881.021257:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876de00.
00000100:00000001:2.0:1787763881.021261:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763881.021263:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.021265:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.021273:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763881.021274:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763881.021276:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763881.021278:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763881.021280:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763881.021283:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876da40 x1874599832669184/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.021288:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763881.021289:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763881.021291:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763881.021292:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763881.021310:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763881.021312:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832669184:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763881.021314:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763881.021316:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763881.021317:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763881.021318:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763881.021321:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763881.021322:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763881.021325:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763881.021327:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787763881.021330:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ece58.
00000100:00000200:2.0:1787763881.021332:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832669184, portal 10
00000100:00000001:2.0:1787763881.021334:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763881.021335:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049524800 : -115096660026816 : ffff9751f876da40)
00000100:00000040:2.0:1787763881.021340:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876da40 x1874599832669184/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763906 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.021345:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763881.021347:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763881.021348:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec990.
00000100:00000200:2.0:1787763881.021350:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832669184, offset 0
00000400:00000200:2.0:1787763881.021352:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763881.021355:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763881.021358:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763881.021377:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763881.021379:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763881.021381:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763881.021383:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763881.021385:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58463:58463:256:4294967295] 192.168.203.110@tcp LPNI seq info [58463:58463:8:4294967295]
00000400:00000200:2.0:1787763881.021389:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763881.021394:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763881.021397:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763881.021400:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f400.
00000800:00000200:2.0:1787763881.021403:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763881.021405:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787763881.021408:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763881.021410:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1787763881.021413:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763881.021414:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
00000400:00000200:2.0:1787763881.021417:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763881.021421:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763881.021441:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec990
00000400:00000010:2.0:1787763881.021443:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec990.
00000100:00000001:2.0:1787763881.021446:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763881.021450:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876da40 x1874599832669184/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763906 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.021468:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763881.021472:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876da40 x1874599832669184/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763906 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.021480:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763881.021482:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763881.021486:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763881.021490:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787763881.021493:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763881.021495:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316268064
00000100:00000001:2.0:1787763881.021498:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316269299
00000100:00000001:2.0:1787763881.021499:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316268064
00000100:00000001:2.0:1787763881.021501:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316269299
00000100:00000001:2.0:1787763881.021503:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763881.021504:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763881.021506:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763881.021509:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763881/real 1787763881]  req@ffff9751f876da40 x1874599832669184/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763881.021515:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763881.021516:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763881.021517:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787763881.021519:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000400:00000010:2.0:1787763881.021520:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000100:00000001:2.0:1787763881.021522:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763881.021524:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876da40 x1874599832669184/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763881.021530:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876da40 x1874599832669184/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.021535:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763881.021536:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.021538:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763881.021539:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763881.021541:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876da40 x1874599832669184/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.021546:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763881.021549:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876da40 x1874599832669184/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.021554:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763881.021555:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.021556:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763881.021557:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.021559:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763881.021568:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763881.021570:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763881.021572:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763881.021573:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763881.021576:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876da40 x1874599832669184/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763881.021599:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832669184:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763881.021604:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763881.021608:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cb40 x1874599832669312/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.021617:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763881.021619:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763881.021622:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763881.021624:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763881.021627:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763881.021632:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832669312:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763881.021636:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763881.021638:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763881.021641:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763881.021643:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763881.021648:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763881.021650:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763881.021655:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763881.021658:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787763881.021661:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ece58.
00000100:00000200:2.0:1787763881.021664:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832669312, portal 10
00000100:00000001:2.0:1787763881.021666:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763881.021667:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049520960 : -115096660030656 : ffff9751f876cb40)
00000100:00000040:2.0:1787763881.021672:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cb40 x1874599832669312/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763906 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.021678:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763881.021680:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763881.021681:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec990.
00000100:00000200:2.0:1787763881.021683:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832669312, offset 0
00000400:00000200:2.0:1787763881.021686:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763881.021689:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763881.021692:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763881.021695:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763881.021697:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763881.021699:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763881.021701:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763881.021704:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58464:58464:256:4294967295] 192.168.203.110@tcp LPNI seq info [58464:58464:8:4294967295]
00000400:00000200:2.0:1787763881.021707:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763881.021728:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763881.021732:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763881.021734:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f400.
00000800:00000200:2.0:1787763881.021738:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763881.021741:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787763881.021744:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763881.021746:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1787763881.021748:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763881.021750:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
00000400:00000200:2.0:1787763881.021752:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763881.021755:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763881.021757:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec990
00000400:00000010:2.0:1787763881.021759:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec990.
00000100:00000001:2.0:1787763881.021760:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763881.021763:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cb40 x1874599832669312/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763906 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.021769:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763881.021771:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cb40 x1874599832669312/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763906 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.021776:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763881.021777:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763881.021779:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763881.021782:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787763881.021784:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763881.021785:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316268052
00000100:00000001:2.0:1787763881.021787:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316269299
00000100:00000001:2.0:1787763881.021789:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316268052
00000100:00000001:2.0:1787763881.021790:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316269299
00000100:00000001:2.0:1787763881.021792:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763881.021794:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763881.021795:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763881.021797:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763881/real 1787763881]  req@ffff9751f876cb40 x1874599832669312/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763881.021803:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763881.021805:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763881.021806:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787763881.021808:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000400:00000010:2.0:1787763881.021809:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000100:00000001:2.0:1787763881.021810:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763881.021828:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cb40 x1874599832669312/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763881.021833:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cb40 x1874599832669312/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.021837:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763881.021838:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.021839:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763881.021840:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763881.021842:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cb40 x1874599832669312/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.021846:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763881.021848:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cb40 x1874599832669312/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.021852:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763881.021854:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.021856:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763881.021856:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.021858:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763881.021859:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763881.021860:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763881.021862:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763881.021863:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763881.021865:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cb40 x1874599832669312/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763881.021874:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832669312:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763881.021877:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763881.021879:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832669440/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.021883:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763881.021884:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763881.021885:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763881.021886:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763881.021887:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763881.021889:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832669440:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763881.021907:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763881.021909:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763881.021910:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763881.021911:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763881.021914:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787763881.021916:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763881.021919:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763881.021921:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787763881.021923:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ece58.
00000100:00000200:2.0:1787763881.021926:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832669440, portal 4
00000100:00000001:2.0:1787763881.021928:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763881.021929:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
00000100:00000040:2.0:1787763881.021933:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832669440/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763906 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.021939:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763881.021940:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763881.021942:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec990.
00000100:00000200:2.0:1787763881.021944:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832669440, offset 0
00000400:00000200:2.0:1787763881.021947:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763881.021950:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763881.021953:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763881.021956:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763881.021958:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763881.021960:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763881.021962:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763881.021964:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58465:58465:256:4294967295] 192.168.203.110@tcp LPNI seq info [58465:58465:8:4294967295]
00000400:00000200:2.0:1787763881.021967:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763881.021972:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763881.021976:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763881.021978:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f400.
00000800:00000200:2.0:1787763881.021980:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763881.021983:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787763881.021986:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763881.021988:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1787763881.021991:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763881.021994:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
00000400:00000200:2.0:1787763881.021997:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763881.022001:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763881.022005:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec990
00000400:00000010:2.0:1787763881.022007:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec990.
00000100:00000001:2.0:1787763881.022010:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763881.022015:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876c000 x1874599832669440/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763906 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.022040:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763881.022043:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832669440/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763906 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.022062:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763881.022064:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763881.022068:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763881.022071:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787763881.022074:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763881.022077:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316268069
00000100:00000001:2.0:1787763881.022078:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316269299
00000100:00000001:2.0:1787763881.022080:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316268069
00000100:00000001:2.0:1787763881.022081:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316269299
00000100:00000001:2.0:1787763881.022083:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763881.022084:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763881.022086:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763881.022089:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763881/real 1787763881]  req@ffff9751f876c000 x1874599832669440/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763881.022094:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763881.022095:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763881.022096:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787763881.022114:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000400:00000010:2.0:1787763881.022115:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000100:00000001:2.0:1787763881.022117:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763881.022119:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832669440/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763881.022126:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832669440/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.022130:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763881.022131:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.022133:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763881.022134:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763881.022136:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832669440/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.022140:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763881.022143:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832669440/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.022148:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763881.022166:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.022168:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763881.022169:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.022170:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763881.022171:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763881.022174:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763881.022176:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763881.022178:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763881.022181:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832669440/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763881.022186:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832669440:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763881.022189:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.022191:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763881.022193:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876da40 x1874599832669184/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.022198:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763881.022199:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763881.022201:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763881.022203:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763881.022204:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763881.022206:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763881.022207:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763881.022209:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017800.
02000000:00000001:2.0:1787763881.022212:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763881.022213:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763881.022214:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876da40.
00000100:00000001:2.0:1787763881.022216:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763881.022217:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.022218:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763881.022220:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cb40 x1874599832669312/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.022233:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763881.022235:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763881.022236:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763881.022239:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763881.022241:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763881.022242:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763881.022244:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763881.022246:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
02000000:00000001:2.0:1787763881.022249:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763881.022250:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763881.022251:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cb40.
00000100:00000001:2.0:1787763881.022255:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763881.022256:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.022258:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763881.022263:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832669440/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763906 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763881.022271:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763881.022273:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763881.022275:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787763881.022277:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763881.022278:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763881.022280:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787763881.022282:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763881.022284:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:2.0:1787763881.022285:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763881.022286:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763881.022287:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:2.0:1787763881.022289:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763881.022290:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.022292:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763881.022295:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763881.022296:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763881.022298:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763881.022299:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763881.660243:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763881.660246:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763881.660251:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763881.660252:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763881.660256:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763881.660257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763881.660260:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763881.660263:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763881.660264:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763881.660268:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763881.660269:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763881.660271:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787763881.660275:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763881.660275:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763881.660281:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787763881.660282:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763881.660283:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763881.660287:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763881.660288:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763881.660289:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763881.660295:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763881.660342:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763881.660346:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763881.660349:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763882.044203:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763882.044210:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763882.044211:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763882.044216:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763882.044219:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763882.044222:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763882.044224:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763882.044226:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763882.044228:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763882.300243:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316269299, at 4316269312
00000100:00000010:3.0:1787763882.300257:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7f00.
00000100:00000040:2.0:1787763882.300297:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316269299, at 4316269312
00000100:00000010:2.0:1787763882.300304:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00000100:00000001:2.0:1787763882.684214:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763882.684221:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763882.684223:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763882.684223:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763882.684227:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763882.684229:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763882.684233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763882.684234:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763882.684235:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763882.684239:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763882.684241:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763882.684243:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763882.684244:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763882.684246:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763882.684250:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763882.684297:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763882.684300:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763882.684303:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763883.068201:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763883.068207:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763883.068210:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763883.068216:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763883.068221:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763883.068224:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763883.068227:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763883.068230:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763883.068234:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763883.708252:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763883.708255:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763883.708259:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763883.708279:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763883.708280:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787763883.708287:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763883.708288:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763883.708298:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00100000:2.0:1787763883.708300:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763883.708302:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763883.708310:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763883.708312:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763883.708314:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763883.708321:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763883.708322:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763883.708326:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763883.708332:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763883.708333:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763883.708338:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763883.708340:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763883.708347:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763883.708427:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763883.708434:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763883.708438:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763884.092264:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763884.092271:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763884.092273:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763884.092278:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763884.092281:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763884.092285:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763884.092287:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763884.092289:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763884.092292:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763884.732236:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763884.732240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763884.732243:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763884.732245:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763884.732247:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763884.732253:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763884.732253:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763884.732256:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763884.732259:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763884.732261:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763884.732264:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763884.732265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763884.732268:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763884.732271:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763884.732274:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763884.732321:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763884.732325:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763884.732328:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763885.116235:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763885.116242:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763885.116244:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763885.116249:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763885.116253:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763885.116256:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763885.116259:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763885.116261:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763885.116264:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763885.756233:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763885.756234:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763885.756240:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763885.756242:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763885.756246:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763885.756247:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763885.756249:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763885.756254:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763885.756254:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763885.756256:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763885.756260:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763885.756264:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763885.756267:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763885.756269:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763885.756272:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763885.756315:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763885.756319:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763885.756321:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763886.140253:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763886.140259:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763886.140262:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763886.140266:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763886.140269:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763886.140273:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763886.140275:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763886.140277:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763886.140279:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763886.140286:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763886.140298:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787763886.140302:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763886.140307:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763886.140312:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763886.140320:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21600
00000400:00000200:1.0:1787763886.140328:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763886.140334:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763886.140338:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763886.140340:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763886.140343:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763886.140345:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763886.140348:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763886.140351:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763886.140352:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787763886.140355:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763886.140358:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787763886.140360:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787763886.140361:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787763886.140364:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763886.140365:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763886.140367:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763886.140368:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763886.140370:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787763886.140373:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763886.140375:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763886.140377:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763886.140381:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763886.140383:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763886.140385:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763886.140388:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763886.140397:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7931680.
00000020:00000040:1.0:1787763886.140400:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787763886.140404:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763886.140405:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763886.140408:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763886.140411:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763886.140415:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763886.140425:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cac00.
00000100:00000001:1.0:1787763886.140432:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763886.140441:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7931680 x1874599832669568/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763886.140451:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763886.140470:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763886.140474:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d4255d6] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7931680 x1874599832669568/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763886.140496:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763886.140499:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763886.140502:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787763886.140503:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763886.140505:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763886.140515:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763886.140519:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21600
00000100:00000001:2.0:1787763886.140521:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:1.0:1787763886.140527:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763886.140527:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763886.140533:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000400:00000200:1.0:1787763886.140539:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763886.140544:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763886.140548:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7931680 x1874599832669568/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787763886.140556:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763886.140577:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763886.140582:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763886.140582:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763886.140591:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763886.140595:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763886.140597:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763886.140600:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00100000:2.0:1787763886.140605:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832669568:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763886.140610:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763886.140614:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:1.0:1787763886.140616:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763886.140619:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
02000000:00000001:2.0:1787763886.140619:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000001:1.0:1787763886.140623:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:2.0:1787763886.140623:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763886.140628:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:1.0:1787763886.140646:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787763886.140649:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000010:2.0:1787763886.140651:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
00000020:00000001:1.0:1787763886.140653:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:2.0:1787763886.140659:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763886.140660:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763886.140666:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763886.140670:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763886.140672:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787763886.140680:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:1.0:1787763886.140681:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787763886.140692:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763886.140696:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763886.140698:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
02000000:00000001:1.0:1787763886.140699:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763886.140707:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763886.140711:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763886.140713:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:2.0:1787763886.140713:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ece58.
00000008:00100000:1.0:1787763886.140719:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763886.140723:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763886.140725:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832669568, portal 25
00000100:00000001:1.0:1787763886.140726:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763886.140736:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000010:1.0:1787763886.140737:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930b40.
00000020:00000040:1.0:1787763886.140745:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000100:00000001:2.0:1787763886.140745:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497727104 : -115097211824512 : ffff9751d7931680)
02000000:00000001:1.0:1787763886.140753:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763886.140756:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763886.140758:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763886.140759:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763886.140761:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763886.140766:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7931680 x1874599832669568/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763911 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1787763886.140767:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9400.
00000100:00000001:1.0:1787763886.140775:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763886.140787:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930b40 x1874599832669696/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763886.140791:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:1.0:1787763886.140800:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:2.0:1787763886.140805:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787763886.140812:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763886.140814:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec990.
00000100:00000040:1.0:1787763886.140820:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022d7f91a] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930b40 x1874599832669696/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763886.140824:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832669568, offset 0
00000400:00000200:2.0:1787763886.140847:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787763886.140850:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763886.140856:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787763886.140862:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:1.0:1787763886.140872:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787763886.140875:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:2.0:1787763886.140879:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00080000:1.0:1787763886.140880:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1787763886.140889:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1787763886.140890:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:2.0:1787763886.140898:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00080000:1.0:1787763886.140916:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21600
00000400:00000200:2.0:1787763886.140918:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763886.140930:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787763886.140932:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763886.140940:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58466:58466:256:4294967295] 192.168.203.110@tcp LPNI seq info [58466:58466:8:4294967295]
00000400:00000200:1.0:1787763886.140943:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763886.140952:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763886.140960:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:2.0:1787763886.140963:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763886.140980:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763886.140984:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763886.140989:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763886.140996:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763886.141001:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787763886.141007:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763886.141011:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000400:00000200:2.0:1787763886.141012:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000020:00000001:1.0:1787763886.141029:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787763886.141037:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787763886.141064:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:2.0:1787763886.141065:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787763886.141074:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000010:2.0:1787763886.141077:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f400.
00000100:00000040:1.0:1787763886.141078:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763886.141085:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763886.141088:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787763886.141093:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000800:00000200:2.0:1787763886.141094:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787763886.141103:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787763886.141105:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
02000000:00000001:1.0:1787763886.141106:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763886.141111:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763886.141116:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763886.141118:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763886.141120:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763886.141127:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787763886.141128:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787763886.141129:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763886.141136:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930000.
00000020:00000040:1.0:1787763886.141138:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763886.141141:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763886.141143:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763886.141146:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:2.1:1787763886.141167:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1787763886.141174:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763886.141177:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
00000400:00000200:2.0:1787763886.141183:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763886.141191:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763886.141197:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec990
00000400:00000010:2.0:1787763886.141199:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec990.
02000000:00000001:1.0:1787763886.141201:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763886.141205:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763886.141205:0:257575:0:(events.c:34:request_out_callback()) Process entered
02000000:00000010:1.0:1787763886.141212:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8400.
00000100:00000200:2.0:1787763886.141213:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7931680 x1874599832669568/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763911 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763886.141221:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763886.141234:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930000 x1874599832669824/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763886.141234:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763886.141245:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7931680 x1874599832669568/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763911 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.141275:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763886.141278:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763886.141278:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787763886.141285:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763886.141286:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763886.141299:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763886.141304:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316274420
00000100:00000040:1.0:1787763886.141306:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6d90274] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930000 x1874599832669824/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1787763886.141311:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00000001:1.0:1787763886.141325:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763886.141326:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000040:1.0:1787763886.141333:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763886.141337:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316274420
00000100:00000040:1.0:1787763886.141338:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787763886.141346:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000010:2.0:1787763886.141346:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7c00.
00000100:00080000:1.0:1787763886.141355:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787763886.141356:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763886.141357:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763886.141358:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763886.141363:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21600
00000100:00000001:2.0:1787763886.141364:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000400:00000200:1.0:1787763886.141369:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763886.141369:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763886.141378:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763886.141380:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787763886.141385:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00080000:1.0:1787763886.141386:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763886.141394:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00100000:2.0:1787763886.141395:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763886/real 1787763886]  req@ffff9751d7931680 x1874599832669568/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763886.141403:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763886.141412:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763886.141420:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763886.141427:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000001:2.0:1787763886.141428:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000040:1.0:1787763886.141432:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000400:00000001:2.0:1787763886.141432:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763886.141437:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000020:00000001:1.0:1787763886.141438:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:2.0:1787763886.141449:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000020:00000040:1.0:1787763886.141450:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787763886.141459:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000010:2.0:1787763886.141460:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000020:00000001:1.0:1787763886.141468:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:2.0:1787763886.141469:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763886.141487:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7931680 x1874599832669568/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787763886.141492:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763886.141501:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763886.141511:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000200:2.0:1787763886.141521:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7931680 x1874599832669568/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763886.141522:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763886.141535:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:2.0:1787763886.141538:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763886.141543:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763886.141548:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00080000:1.0:1787763886.141549:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763886.141550:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763886.141560:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763886.141561:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7931680 x1874599832669568/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787763886.141570:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787763886.141573:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:1.0:1787763886.141578:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1787763886.141578:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7931680 x1874599832669568/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787763886.141591:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763886.141602:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763886.141608:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:2.0:1787763886.141610:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000008:00000001:1.0:1787763886.141615:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763886.141617:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763886.141619:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763886.141621:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763886.141624:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:1.0:1787763886.141626:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933480.
00000100:00000001:2.0:1787763886.141627:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000020:00000040:1.0:1787763886.141635:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:2.0:1787763886.141636:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:1.0:1787763886.141650:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00080000:2.0:1787763886.141650:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000001:1.0:1787763886.141656:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763886.141661:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00080000:2.0:1787763886.141663:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
02000000:00000001:1.0:1787763886.141670:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787763886.141671:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
02000000:00000001:1.0:1787763886.141673:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763886.141681:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7931680 x1874599832669568/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1787763886.141682:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca000.
00000100:00000001:1.0:1787763886.141692:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763886.141705:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832669568:192.168.203.110@tcp:250:kworker.0
00000100:00100000:1.0:1787763886.141706:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933480 x1874599832669952/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763886.141721:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763886.141728:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763886.141729:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787763886.141735:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763886.141739:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7931680 x1874599832669568/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787763886.141747:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3c2c33c] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933480 x1874599832669952/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.141757:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763886.141763:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763886.141769:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
00000100:00000001:1.0:1787763886.141779:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763886.141781:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000040:1.0:1787763886.141791:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21610)
00000020:00000001:2.0:1787763886.141791:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763886.141798:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787763886.141802:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763886.141805:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cac00.
02000000:00000001:2.0:1787763886.141811:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763886.141813:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763886.141816:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7931680.
00000100:00000001:2.0:1787763886.141821:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763886.141823:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763886.141826:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763886.141834:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763886.141836:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763886.141838:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763886.141840:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763886.141841:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763886.141845:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930b40 x1874599832669696/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.141851:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763886.141852:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763886.141862:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763886.141864:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763886.141866:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763886.141868:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832669696:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763886.141870:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763886.141872:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763886.141874:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763886.141875:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763886.141878:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763886.141879:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763886.141882:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763886.141884:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787763886.141887:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ece58.
00000100:00000200:2.0:1787763886.141889:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832669696, portal 10
00000100:00000001:2.0:1787763886.141894:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763886.141895:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497724224 : -115097211827392 : ffff9751d7930b40)
00000100:00000040:2.0:1787763886.141912:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930b40 x1874599832669696/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763911 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.141920:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763886.141922:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763886.141924:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec990.
00000100:00000200:2.0:1787763886.141927:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832669696, offset 0
00000400:00000200:2.0:1787763886.141931:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763886.141936:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763886.141941:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763886.141944:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763886.141947:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763886.141950:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763886.141952:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763886.141956:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58467:58467:256:4294967295] 192.168.203.110@tcp LPNI seq info [58467:58467:8:4294967295]
00000400:00000200:2.0:1787763886.141961:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763886.141978:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763886.141982:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763886.141987:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f400.
00000800:00000200:2.0:1787763886.141991:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763886.141995:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787763886.142001:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763886.142004:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1787763886.142009:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763886.142011:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
00000400:00000200:2.0:1787763886.142026:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763886.142029:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763886.142031:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec990
00000400:00000010:2.0:1787763886.142032:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec990.
00000100:00000001:2.0:1787763886.142035:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763886.142037:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930b40 x1874599832669696/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763911 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142054:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763886.142056:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930b40 x1874599832669696/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763911 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142065:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763886.142066:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763886.142068:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763886.142072:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787763886.142074:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763886.142076:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316273184
00000100:00000001:2.0:1787763886.142078:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316274420
00000100:00000001:2.0:1787763886.142080:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316273184
00000100:00000001:2.0:1787763886.142081:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316274420
00000100:00000001:2.0:1787763886.142082:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763886.142084:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763886.142085:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763886.142088:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763886/real 1787763886]  req@ffff9751d7930b40 x1874599832669696/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763886.142093:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763886.142094:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763886.142096:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787763886.142097:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000400:00000010:2.0:1787763886.142098:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000100:00000001:2.0:1787763886.142101:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763886.142104:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930b40 x1874599832669696/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763886.142112:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930b40 x1874599832669696/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142119:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763886.142121:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763886.142123:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763886.142124:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763886.142128:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930b40 x1874599832669696/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142157:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763886.142161:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930b40 x1874599832669696/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142166:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763886.142168:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763886.142169:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763886.142170:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763886.142171:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763886.142181:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763886.142183:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763886.142185:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763886.142187:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763886.142189:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930b40 x1874599832669696/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763886.142194:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832669696:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763886.142197:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763886.142199:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930000 x1874599832669824/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142204:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763886.142205:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763886.142206:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763886.142208:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763886.142209:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763886.142211:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832669824:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763886.142213:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763886.142215:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763886.142217:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763886.142218:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763886.142221:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763886.142222:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763886.142224:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763886.142226:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787763886.142228:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ece58.
00000100:00000200:2.0:1787763886.142231:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832669824, portal 10
00000100:00000001:2.0:1787763886.142232:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763886.142233:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497721344 : -115097211830272 : ffff9751d7930000)
00000100:00000040:2.0:1787763886.142237:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930000 x1874599832669824/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763911 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142242:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763886.142243:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763886.142245:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec990.
00000100:00000200:2.0:1787763886.142246:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832669824, offset 0
00000400:00000200:2.0:1787763886.142249:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763886.142252:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763886.142254:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763886.142256:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763886.142258:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763886.142260:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763886.142261:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763886.142264:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58468:58468:256:4294967295] 192.168.203.110@tcp LPNI seq info [58468:58468:8:4294967295]
00000400:00000200:2.0:1787763886.142266:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763886.142271:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763886.142273:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763886.142275:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f400.
00000800:00000200:2.0:1787763886.142278:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763886.142280:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787763886.142283:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763886.142285:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1787763886.142287:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763886.142289:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
00000400:00000200:2.0:1787763886.142292:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763886.142294:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763886.142296:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec990
00000400:00000010:2.0:1787763886.142297:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec990.
00000100:00000001:2.0:1787763886.142299:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763886.142301:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930000 x1874599832669824/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763911 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142306:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763886.142308:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930000 x1874599832669824/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763911 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142312:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763886.142314:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763886.142315:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763886.142318:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787763886.142319:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763886.142321:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316273173
00000100:00000001:2.0:1787763886.142322:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316274420
00000100:00000001:2.0:1787763886.142324:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316273173
00000100:00000001:2.0:1787763886.142325:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316274420
00000100:00000001:2.0:1787763886.142327:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763886.142328:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763886.142329:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763886.142331:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763886/real 1787763886]  req@ffff9751d7930000 x1874599832669824/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763886.142336:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763886.142337:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763886.142338:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787763886.142339:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000400:00000010:2.0:1787763886.142340:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000100:00000001:2.0:1787763886.142342:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763886.142344:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930000 x1874599832669824/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763886.142349:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930000 x1874599832669824/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142353:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763886.142354:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763886.142355:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763886.142356:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763886.142358:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930000 x1874599832669824/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142362:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763886.142364:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930000 x1874599832669824/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142368:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763886.142370:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763886.142371:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763886.142372:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763886.142373:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763886.142374:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763886.142376:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763886.142377:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763886.142379:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763886.142381:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930000 x1874599832669824/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763886.142386:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832669824:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763886.142388:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763886.142391:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933480 x1874599832669952/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142395:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763886.142396:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763886.142398:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763886.142399:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763886.142400:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763886.142402:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832669952:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763886.142404:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763886.142405:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763886.142406:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763886.142407:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763886.142410:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787763886.142411:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763886.142413:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763886.142415:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787763886.142416:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ece58.
00000100:00000200:2.0:1787763886.142418:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832669952, portal 4
00000100:00000001:2.0:1787763886.142420:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763886.142421:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497734784 : -115097211816832 : ffff9751d7933480)
00000100:00000040:2.0:1787763886.142424:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933480 x1874599832669952/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763911 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142428:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763886.142430:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763886.142431:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec990.
00000100:00000200:2.0:1787763886.142433:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832669952, offset 0
00000400:00000200:2.0:1787763886.142435:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763886.142437:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763886.142439:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763886.142440:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763886.142442:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763886.142443:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763886.142445:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763886.142448:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58469:58469:256:4294967295] 192.168.203.110@tcp LPNI seq info [58469:58469:8:4294967295]
00000400:00000200:2.0:1787763886.142451:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763886.142454:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763886.142457:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763886.142459:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f400.
00000800:00000200:2.0:1787763886.142461:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763886.142463:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787763886.142465:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763886.142467:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1787763886.142480:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763886.142481:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
00000400:00000200:2.0:1787763886.142483:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763886.142485:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763886.142487:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec990
00000400:00000010:2.0:1787763886.142488:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec990.
00000100:00000001:2.0:1787763886.142490:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763886.142492:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933480 x1874599832669952/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763911 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142497:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763886.142499:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933480 x1874599832669952/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763911 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142503:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763886.142504:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763886.142506:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763886.142507:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787763886.142509:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763886.142510:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316273190
00000100:00000001:2.0:1787763886.142512:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316274420
00000100:00000001:2.0:1787763886.142513:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316273190
00000100:00000001:2.0:1787763886.142514:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316274420
00000100:00000001:2.0:1787763886.142516:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763886.142518:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763886.142519:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763886.142521:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763886/real 1787763886]  req@ffff9751d7933480 x1874599832669952/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763886.142526:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763886.142527:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763886.142528:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787763886.142529:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000400:00000010:2.0:1787763886.142530:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000100:00000001:2.0:1787763886.142531:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763886.142533:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933480 x1874599832669952/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763886.142538:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933480 x1874599832669952/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142542:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763886.142543:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763886.142544:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763886.142545:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763886.142547:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933480 x1874599832669952/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142551:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763886.142553:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933480 x1874599832669952/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142557:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763886.142558:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763886.142559:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763886.142560:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763886.142561:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763886.142562:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763886.142563:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763886.142566:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763886.142567:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763886.142570:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933480 x1874599832669952/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763886.142575:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832669952:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763886.142577:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763886.142578:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763886.142580:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930b40 x1874599832669696/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142584:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763886.142585:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763886.142586:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763886.142588:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763886.142589:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763886.142589:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763886.142591:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763886.142592:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9400.
02000000:00000001:2.0:1787763886.142594:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763886.142594:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763886.142595:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930b40.
00000100:00000001:2.0:1787763886.142597:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763886.142598:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763886.142599:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763886.142601:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930000 x1874599832669824/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142605:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763886.142606:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763886.142607:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763886.142608:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763886.142609:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763886.142610:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763886.142611:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763886.142612:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8400.
02000000:00000001:2.0:1787763886.142614:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763886.142614:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763886.142615:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930000.
00000100:00000001:2.0:1787763886.142617:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763886.142618:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763886.142619:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763886.142620:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933480 x1874599832669952/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763911 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763886.142624:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763886.142625:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763886.142626:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787763886.142627:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763886.142627:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763886.142628:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787763886.142629:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763886.142630:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca000.
02000000:00000001:2.0:1787763886.142631:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763886.142632:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763886.142633:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933480.
00000100:00000001:2.0:1787763886.142635:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763886.142636:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763886.142637:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763886.142639:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763886.142640:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763886.142642:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763886.142643:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763886.780261:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763886.780268:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763886.780269:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763886.780271:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763886.780275:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763886.780277:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763886.780284:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763886.780285:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763886.780294:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763886.780294:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763886.780301:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787763886.780302:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763886.780309:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763886.780310:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763886.780312:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763886.780315:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763886.780318:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763886.780321:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763886.780325:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763886.780329:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763886.780331:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763886.780383:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763886.780387:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763886.780390:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763887.164273:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763887.164280:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763887.164282:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763887.164287:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763887.164292:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763887.164295:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763887.164298:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763887.164300:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763887.164303:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763887.420235:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316274420, at 4316274432
00000100:00000040:2.0:1787763887.420242:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316274420, at 4316274432
00000100:00000010:3.0:1787763887.420255:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7c00.
00000100:00000010:2.0:1787763887.420259:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00000100:00000001:2.0:1787763887.804201:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763887.804207:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763887.804209:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763887.804215:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763887.804219:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763887.804222:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763887.804224:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763887.804226:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763887.804229:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763887.804249:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763887.804266:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763887.804271:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763887.804273:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763887.804276:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763887.804277:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763887.804371:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763887.804376:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763887.804379:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763888.188205:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763888.188213:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763888.188215:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763888.188219:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763888.188222:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763888.188225:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763888.188234:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763888.188236:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763888.188238:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763888.828210:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763888.828217:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763888.828218:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763888.828223:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763888.828225:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763888.828228:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763888.828230:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763888.828234:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763888.828239:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763888.828240:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763888.828244:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763888.828250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763888.828251:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763888.828254:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763888.828260:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763888.828306:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763888.828310:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763888.828314:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763889.212239:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763889.212246:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763889.212247:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763889.212252:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763889.212255:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763889.212258:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763889.212260:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763889.212262:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763889.212265:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763889.852243:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763889.852247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763889.852249:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763889.852253:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763889.852254:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763889.852258:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763889.852262:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763889.852263:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763889.852265:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763889.852267:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763889.852268:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763889.852272:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763889.852275:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763889.852276:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763889.852280:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763889.852351:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763889.852356:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763889.852359:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763890.236220:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763890.236227:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763890.236229:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763890.236234:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763890.236238:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763890.236241:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763890.236243:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763890.236245:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763890.236248:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763890.876240:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763890.876242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763890.876250:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763890.876254:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763890.876259:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763890.876261:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763890.876266:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763890.876267:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763890.876269:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763890.876271:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763890.876273:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763890.876275:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763890.876275:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763890.876279:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763890.876283:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763890.876339:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763890.876345:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763890.876348:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763891.260250:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763891.260258:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763891.260260:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763891.260265:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763891.260269:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763891.260272:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763891.260274:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763891.260276:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763891.260279:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763891.260287:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787763891.260293:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787763891.260295:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763891.260298:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763891.260302:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787763891.260308:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21605
00000400:00000200:3.0:1787763891.260316:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763891.260322:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763891.260324:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763891.260327:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763891.260329:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763891.260331:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763891.260333:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763891.260336:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763891.260338:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787763891.260340:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763891.260344:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787763891.260346:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787763891.260348:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787763891.260350:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763891.260351:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763891.260353:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763891.260355:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763891.260357:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787763891.260360:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763891.260362:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763891.260364:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763891.260367:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763891.260369:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763891.260370:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787763891.260372:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763891.260380:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bde00.
00000020:00000040:3.0:1787763891.260382:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787763891.260385:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763891.260386:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763891.260388:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763891.260391:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763891.260394:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763891.260400:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613800.
00000100:00000001:3.0:1787763891.260405:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763891.260412:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bde00 x1874599832670080/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787763891.260420:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787763891.260423:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763891.260427:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000941163c2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bde00 x1874599832670080/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763891.260442:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763891.260444:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787763891.260446:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787763891.260448:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763891.260449:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763891.260451:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787763891.260453:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21605
00000400:00000200:3.0:1787763891.260456:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763891.260458:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763891.260460:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763891.260462:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763891.260463:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763891.260464:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763891.260467:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1787763891.260468:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763891.260471:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1787763891.260472:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763891.260475:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763891.260477:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:3.0:1787763891.260479:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:2.0:1787763891.260482:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bde00 x1874599832670080/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787763891.260482:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:3.0:1787763891.260486:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:3.0:1787763891.260490:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:3.0:1787763891.260497:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763891.260502:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763891.260502:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763891.260505:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763891.260507:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763891.260509:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:2.0:1787763891.260510:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:3.0:1787763891.260511:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:2.0:1787763891.260515:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:3.0:1787763891.260515:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:2.0:1787763891.260520:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763891.260521:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763891.260525:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:2.0:1787763891.260528:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832670080:192.168.203.110@tcp:250:kworker.0
02000000:00000001:3.0:1787763891.260531:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763891.260532:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763891.260536:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763891.260538:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1787763891.260539:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1787763891.260543:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1787763891.260544:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1787763891.260546:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:3.0:1787763891.260549:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763891.260552:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763891.260553:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763891.260556:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763891.260556:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bed00.
00000020:00000040:3.0:1787763891.260561:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787763891.260564:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:2.0:1787763891.260565:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:3.0:1787763891.260566:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763891.260570:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:2.0:1787763891.260571:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
02000000:00000001:3.0:1787763891.260571:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763891.260573:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763891.260578:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610400.
00000400:00000010:2.0:1787763891.260580:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ece58.
00000100:00000200:2.0:1787763891.260587:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832670080, portal 25
00000100:00000001:2.0:1787763891.260589:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763891.260590:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049853440 : -115096659698176 : ffff9751f87bde00)
00000100:00000001:3.0:1787763891.260591:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763891.260596:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bde00 x1874599832670080/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763916 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763891.260598:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bed00 x1874599832670208/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763891.260614:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1787763891.260618:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:2.0:1787763891.260622:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787763891.260623:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763891.260627:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec990.
00000100:00000200:2.0:1787763891.260631:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832670080, offset 0
00000100:00000040:3.0:1787763891.260633:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000079751c9a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bed00 x1874599832670208/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787763891.260639:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787763891.260647:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763891.260649:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:3.0:1787763891.260649:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787763891.260655:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1787763891.260657:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763891.260659:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1787763891.260660:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:3.0:1787763891.260662:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787763891.260666:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21605
00000400:00000200:2.0:1787763891.260667:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763891.260673:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763891.260676:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763891.260678:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763891.260681:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58470:58470:256:4294967295] 192.168.203.110@tcp LPNI seq info [58470:58470:8:4294967295]
00000400:00000200:2.0:1787763891.260684:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763891.260690:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000400:00000200:3.0:1787763891.260696:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:2.0:1787763891.260697:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763891.260701:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000800:00000010:2.0:1787763891.260702:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f400.
00000100:00080000:3.0:1787763891.260703:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763891.260708:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763891.260710:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1787763891.260714:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:3.0:1787763891.260714:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763891.260720:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763891.260722:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:2.1:1787763891.260724:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:3.0:1787763891.260725:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:3.0:1787763891.260729:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763891.260731:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:3.0:1787763891.260733:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:3.0:1787763891.260734:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:3.0:1787763891.260737:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763891.260738:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763891.260740:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763891.260742:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763891.260742:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763891.260743:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:3.0:1787763891.260749:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:2.0:1787763891.260750:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763891.260753:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763891.260754:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316279539
02000000:00000001:3.0:1787763891.260756:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763891.260760:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000010:2.0:1787763891.260763:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
02000000:00000001:3.0:1787763891.260764:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763891.260768:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787763891.260771:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1787763891.260772:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000008:00000001:3.0:1787763891.260773:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763891.260777:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763891.260782:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316279539
00000100:00000010:3.0:1787763891.260782:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87be1c0.
00000020:00000040:3.0:1787763891.260784:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787763891.260788:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000010:2.0:1787763891.260790:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb75a0.
02000000:00000001:3.0:1787763891.260790:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763891.260792:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763891.260794:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787763891.260795:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
02000000:00000001:3.0:1787763891.260796:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763891.260801:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000010:3.0:1787763891.260803:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613c00.
00000100:00000001:2.0:1787763891.260804:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763891.260809:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763891.260809:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763891.260815:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763891.260819:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87be1c0 x1874599832670336/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763891.260823:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763891.260825:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763891.260826:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1787763891.260829:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763891.260831:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763891.260835:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1787763891.260835:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e2011c6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87be1c0 x1874599832670336/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763891.260838:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763891.260845:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bed00 x1874599832670208/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763891.260847:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763891.260852:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763891.260858:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763891.260859:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763891.260863:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
02000000:00000001:2.0:1787763891.260864:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763891.260868:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:3.0:1787763891.260869:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:2.0:1787763891.260873:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763891.260874:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787763891.260878:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:2.0:1787763891.260880:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832670208:192.168.203.110@tcp:38:kworker.0
00000100:00080000:3.0:1787763891.260881:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763891.260887:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763891.260888:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:3.0:1787763891.260891:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21605
02000000:00000001:2.0:1787763891.260892:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763891.260895:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763891.260900:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000400:00000200:3.0:1787763891.260900:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000010:2.0:1787763891.260905:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
00000400:00000200:3.0:1787763891.260905:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763891.260908:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
02000000:00000001:2.0:1787763891.260909:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763891.260914:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:2.0:1787763891.260915:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763891.260919:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1787763891.260920:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000100:00000040:3.0:1787763891.260924:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000010:2.0:1787763891.260927:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000001:3.0:1787763891.260927:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763891.260932:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000200:2.0:1787763891.260933:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832670208, portal 10
00000020:00000040:3.0:1787763891.260934:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:3.0:1787763891.260938:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1787763891.260939:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000020:00000040:3.0:1787763891.260940:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000001:2.0:1787763891.260941:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049857280 : -115096659694336 : ffff9751f87bed00)
00000020:00000001:3.0:1787763891.260943:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:3.0:1787763891.260946:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:3.0:1787763891.260950:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000040:2.0:1787763891.260952:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bed00 x1874599832670208/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763916 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763891.260953:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763891.260957:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763891.260961:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763891.260965:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:2.0:1787763891.260969:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:3.0:1787763891.260970:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763891.260975:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763891.260976:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
02000000:00000001:3.0:1787763891.260990:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763891.260993:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:2.0:1787763891.260996:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eccc0.
02000000:00000001:3.0:1787763891.260998:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763891.261000:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832670208, offset 0
00000100:00000001:3.0:1787763891.261001:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787763891.261005:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000400:00000200:2.0:1787763891.261009:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000008:00000001:3.0:1787763891.261010:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763891.261013:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763891.261016:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000010:3.0:1787763891.261018:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc000.
00000400:00000200:2.0:1787763891.261022:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000020:00000040:3.0:1787763891.261023:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000400:00000200:2.0:1787763891.261027:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:3.0:1787763891.261027:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763891.261030:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763891.261032:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:3.0:1787763891.261033:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763891.261035:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763891.261038:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763891.261040:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
02000000:00000010:3.0:1787763891.261042:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611800.
00000400:00000200:2.0:1787763891.261047:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:3.0:1787763891.261048:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763891.261055:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc000 x1874599832670464/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787763891.261057:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58471:58471:255:4294967295] 192.168.203.110@tcp LPNI seq info [58471:58471:7:4294967295]
00000100:00000001:3.0:1787763891.261070:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1787763891.261071:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763891.261072:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763891.261078:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3fc1c6c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc000 x1874599832670464/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787763891.261089:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1787763891.261096:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787763891.261102:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000040:3.0:1787763891.261105:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21615)
00000800:00000010:2.0:1787763891.261109:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646ff00.
00000800:00000200:2.0:1787763891.261115:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763891.261121:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787763891.261125:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763891.261129:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787763891.261156:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763891.261159:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316278304
00000100:00000001:2.0:1787763891.261162:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316279539
00000100:00000001:2.0:1787763891.261165:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316278304
00000100:00000001:2.0:1787763891.261167:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316279539
00000100:00000001:2.0:1787763891.261170:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763891.261172:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763891.261175:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763891.261177:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763891.261181:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763891.261182:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1787763891.261184:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763891.261186:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763891.261187:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763891.261192:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87be1c0 x1874599832670336/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763891.261200:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763891.261201:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763891.261203:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763891.261205:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763891.261208:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763891.261211:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832670336:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763891.261216:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763891.261218:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763891.261219:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763891.261222:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763891.261226:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763891.261228:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763891.261231:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763891.261234:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da04d0.
00000400:00000010:2.0:1787763891.261237:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763891.261239:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832670336, portal 10
00000100:00000001:2.0:1787763891.261241:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763891.261242:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049854400 : -115096659697216 : ffff9751f87be1c0)
00000100:00000040:2.0:1787763891.261247:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87be1c0 x1874599832670336/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763916 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763891.261252:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763891.261253:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763891.261255:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec770.
00000100:00000200:2.0:1787763891.261256:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832670336, offset 0
00000400:00000200:2.0:1787763891.261265:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763891.261268:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763891.261271:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763891.261274:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763891.261277:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763891.261279:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763891.261282:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763891.261285:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58472:58472:254:4294967295] 192.168.203.110@tcp LPNI seq info [58472:58472:6:4294967295]
00000400:00000200:2.0:1787763891.261289:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763891.261296:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763891.261301:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763891.261305:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763891.261309:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763891.261313:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787763891.261316:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763891.261319:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787763891.261323:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763891.261325:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316278292
00000100:00000001:2.0:1787763891.261327:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316279539
00000100:00000001:2.0:1787763891.261329:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316278292
00000100:00000001:2.0:1787763891.261331:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316279539
00000100:00000001:2.0:1787763891.261333:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763891.261335:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763891.261337:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763891.261341:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc000 x1874599832670464/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763891.261349:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763891.261350:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763891.261352:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763891.261354:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763891.261356:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763891.261359:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832670464:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763891.261363:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763891.261365:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763891.261366:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763891.261369:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763891.261373:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763891.261374:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763891.261378:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763891.261381:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763891.261384:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000200:2.0:1787763891.261387:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832670464, portal 4
00000100:00000001:2.0:1787763891.261388:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763891.261389:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049845760 : -115096659705856 : ffff9751f87bc000)
00000100:00000040:2.0:1787763891.261394:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc000 x1874599832670464/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763916 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763891.261399:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763891.261402:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763891.261405:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec440.
00000100:00000200:2.0:1787763891.261407:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832670464, offset 0
00000400:00000200:2.0:1787763891.261412:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763891.261416:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763891.261420:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763891.261436:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763891.261439:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763891.261442:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763891.261444:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763891.261447:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58473:58473:253:4294967295] 192.168.203.110@tcp LPNI seq info [58473:58473:5:4294967295]
00000400:00000200:2.0:1787763891.261451:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763891.261457:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763891.261462:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763891.261465:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f300.
00000800:00000200:2.0:1787763891.261471:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763891.261475:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787763891.261478:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763891.261482:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787763891.261485:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763891.261487:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316278309
00000100:00000001:2.0:1787763891.261490:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316279539
00000100:00000001:2.0:1787763891.261492:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316278309
00000100:00000001:2.0:1787763891.261494:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316279539
00000100:00000001:2.0:1787763891.261496:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763891.261499:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763891.261501:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763891.261503:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763891.261507:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763891.261509:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1787763891.261512:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763891.261513:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763891.261515:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763891.261517:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763891.900208:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763891.900212:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763891.900214:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763891.900216:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763891.900218:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763891.900221:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763891.900223:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763891.900224:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763891.900229:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763891.900229:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763891.900233:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763891.900234:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763891.900236:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787763891.900238:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763891.900241:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763891.900241:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763891.900245:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787763891.900246:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763891.900247:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763891.900250:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763891.900252:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763891.900299:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763891.900303:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763891.900305:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763892.284218:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763892.284225:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763892.284228:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763892.284234:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763892.284238:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763892.284242:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763892.284245:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763892.284248:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763892.284251:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763892.540232:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316279539, at 4316279552
00000100:00000010:3.0:1787763892.540246:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb75a0.
00000100:00000040:2.0:1787763892.540272:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316279539, at 4316279552
00000100:00000010:2.0:1787763892.540280:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00000100:00000001:2.0:1787763892.924244:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763892.924248:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763892.924257:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763892.924258:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763892.924266:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763892.924270:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763892.924275:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763892.924277:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763892.924286:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763892.924287:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763892.924293:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763892.924297:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763892.924304:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763892.924307:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763892.924310:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763892.924404:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763892.924408:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763892.924411:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763893.308208:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763893.308215:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763893.308216:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763893.308221:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763893.308224:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763893.308227:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763893.308229:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763893.308231:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763893.308233:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763893.948222:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763893.948229:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763893.948231:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763893.948235:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763893.948239:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763893.948242:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763893.948243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763893.948245:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763893.948255:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763893.948255:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763893.948264:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763893.948264:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763893.948272:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763893.948278:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763893.948281:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763893.948283:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763893.948285:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763893.948290:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763893.948291:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763893.948294:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763893.948294:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763893.948374:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763893.948378:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763893.948381:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763894.332215:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763894.332223:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763894.332226:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763894.332232:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763894.332236:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763894.332240:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763894.332242:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763894.332245:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763894.332247:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:3.0:1787763894.332387:0:257564:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.110:988
00000400:00000100:3.0:1787763894.332429:0:257564:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.110@tcp at host 192.168.203.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:3.0:1787763894.332439:0:257564:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763894.332444:0:257564:0:(router.c:1791:lnet_notify()) 192.168.203.10@tcp notifying 192.168.203.110@tcp: down
00000400:00000200:3.0:1787763894.332448:0:257564:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763894.332453:0:257564:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:3.0:1787763894.332458:0:257564:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:3.0:1787763894.332462:0:257564:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
00000400:00000200:3.0:1787763894.332468:0:257564:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:3.0:1787763894.332473:0:257564:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:3.0:1787763894.332485:0:257564:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec990
00000400:00000010:3.0:1787763894.332489:0:257564:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec990.
00000100:00000001:3.0:1787763894.332495:0:257564:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763894.332505:0:257564:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f87bde00 x1874599832670080/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763916 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763894.332530:0:257564:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763894.332535:0:257564:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bde00 x1874599832670080/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763916 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763894.332544:0:257564:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763894.332547:0:257564:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763894.332561:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763894.332566:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000800:00000100:3.0:1787763894.332567:0:257564:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000100:00000001:2.0:1787763894.332572:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000800:00000010:3.0:1787763894.332573:0:257564:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646ff00.
00000400:00000200:3.0:1787763894.332579:0:257564:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00100000:2.0:1787763894.332585:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763891/real 1787763894]  req@ffff9751f87bde00 x1874599832670080/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787763894.332585:0:257564:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:3.0:1787763894.332593:0:257564:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eccc0
00000400:00000010:3.0:1787763894.332597:0:257564:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eccc0.
00000100:00000001:3.0:1787763894.332604:0:257564:0:(events.c:34:request_out_callback()) Process entered
00000400:00000001:2.0:1787763894.332607:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763894.332612:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000200:3.0:1787763894.332613:0:257564:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f87bed00 x1874599832670208/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763916 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1787763894.332617:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787763894.332623:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000400:00000010:2.0:1787763894.332626:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000100:00000001:2.0:1787763894.332630:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:3.0:1787763894.332633:0:257564:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000200:2.0:1787763894.332637:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bde00 x1874599832670080/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787763894.332643:0:257564:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bed00 x1874599832670208/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763916 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763894.332672:0:257564:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763894.332673:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bde00 x1874599832670080/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763894.332676:0:257564:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763894.332685:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000800:00000100:3.0:1787763894.332687:0:257564:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000100:00000001:2.0:1787763894.332689:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000800:00000010:3.0:1787763894.332692:0:257564:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000100:00000001:2.0:1787763894.332694:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763894.332699:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:3.0:1787763894.332700:0:257564:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00100000:2.0:1787763894.332707:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bde00 x1874599832670080/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787763894.332709:0:257564:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:3.0:1787763894.332717:0:257564:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec770
00000100:00000001:2.0:1787763894.332722:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000400:00000010:3.0:1787763894.332722:0:257564:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec770.
00000100:00000001:3.0:1787763894.332727:0:257564:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:2.0:1787763894.332733:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bde00 x1874599832670080/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763894.332737:0:257564:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f87be1c0 x1874599832670336/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763916 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763894.332751:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763894.332758:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763894.332758:0:257564:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787763894.332762:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763894.332765:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763894.332766:0:257564:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87be1c0 x1874599832670336/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763916 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763894.332769:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763894.332773:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763894.332780:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:3.0:1787763894.332782:0:257564:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763894.332786:0:257564:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:2.0:1787763894.332788:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763894.332792:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000800:00000100:3.0:1787763894.332797:0:257564:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000100:00000040:2.0:1787763894.332802:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bde00 x1874599832670080/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:3.0:1787763894.332803:0:257564:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f300.
00000400:00000200:3.0:1787763894.332810:0:257564:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00100000:2.0:1787763894.332821:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832670080:192.168.203.110@tcp:250:kworker.0
00000400:00000200:3.0:1787763894.332821:0:257564:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:3.0:1787763894.332829:0:257564:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec440
00000100:00000001:2.0:1787763894.332831:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787763894.332834:0:257564:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec440.
00000100:00000001:2.0:1787763894.332835:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1787763894.332841:0:257564:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:2.0:1787763894.332843:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bde00 x1874599832670080/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763894.332851:0:257564:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f87bc000 x1874599832670464/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763916 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763894.332881:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
00000100:00000001:3.0:1787763894.332882:0:257564:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1787763894.332886:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000040:3.0:1787763894.332890:0:257564:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc000 x1874599832670464/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763916 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1787763894.332892:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763894.332899:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000001:3.0:1787763894.332904:0:257564:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763894.332905:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000100:00000001:3.0:1787763894.332909:0:257564:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000040:2.0:1787763894.332910:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787763894.332914:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763894.332917:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613800.
02000000:00000001:2.0:1787763894.332922:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763894.332924:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763894.332925:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bde00.
00000100:00000001:2.0:1787763894.332929:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763894.332931:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763894.332934:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763894.332936:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763894.332938:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763894.332939:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763894.332944:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763891/real 1787763894]  req@ffff9751f87be1c0 x1874599832670336/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763894.332953:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763894.332954:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763894.332973:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da04d0.
00000400:00000200:2.0:1787763894.332976:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763894.332978:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763894.332980:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763894.332983:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87be1c0 x1874599832670336/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763894.332992:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87be1c0 x1874599832670336/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763894.332998:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763894.332999:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763894.333001:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763894.333003:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763894.333007:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87be1c0 x1874599832670336/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763894.333026:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763894.333030:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87be1c0 x1874599832670336/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763894.333037:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763894.333039:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763894.333040:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763894.333042:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763894.333044:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763894.333046:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763894.333048:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763894.333052:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763894.333054:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763894.333058:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87be1c0 x1874599832670336/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763894.333066:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832670336:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763894.333071:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763894.333075:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763891/real 1787763894]  req@ffff9751f87bc000 x1874599832670464/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763894.333083:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763894.333085:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763894.333087:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763894.333091:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:2.0:1787763894.333092:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:2.0:1787763894.333095:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763894.333099:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc000 x1874599832670464/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763894.333109:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc000 x1874599832670464/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763894.333116:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763894.333118:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763894.333119:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763894.333120:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763894.333124:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc000 x1874599832670464/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763894.333130:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763894.333157:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc000 x1874599832670464/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763894.333165:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763894.333168:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763894.333170:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763894.333171:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763894.333173:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763894.333175:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763894.333178:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763894.333181:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763894.333183:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763894.333186:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc000 x1874599832670464/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763894.333192:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832670464:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763894.333196:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763894.333205:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763891/real 1787763894]  req@ffff9751f87bed00 x1874599832670208/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763894.333221:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763894.333222:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763894.333224:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763894.333227:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763894.333229:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:2.0:1787763894.333232:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763894.333235:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bed00 x1874599832670208/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763894.333245:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bed00 x1874599832670208/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763894.333252:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763894.333254:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763894.333256:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763894.333257:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763894.333261:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bed00 x1874599832670208/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763894.333274:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763894.333278:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bed00 x1874599832670208/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763894.333286:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763894.333289:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763894.333291:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763894.333292:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763894.333295:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763894.333311:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763894.333315:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763894.333319:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763894.333323:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763894.333329:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bed00 x1874599832670208/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763894.333338:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832670208:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763894.333344:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763894.333346:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763894.333351:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87be1c0 x1874599832670336/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763894.333359:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763894.333361:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763894.333364:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763894.333368:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763894.333369:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763894.333371:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763894.333373:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763894.333376:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613c00.
02000000:00000001:2.0:1787763894.333379:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763894.333381:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763894.333383:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87be1c0.
00000100:00000001:2.0:1787763894.333386:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763894.333388:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763894.333390:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763894.333394:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc000 x1874599832670464/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763894.333401:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763894.333402:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763894.333404:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763894.333406:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763894.333407:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763894.333408:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787763894.333409:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763894.333410:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611800.
02000000:00000001:2.0:1787763894.333411:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763894.333413:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763894.333414:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc000.
00000100:00000001:2.0:1787763894.333416:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763894.333417:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763894.333418:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763894.333420:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bed00 x1874599832670208/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763916 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763894.333424:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763894.333425:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763894.333426:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763894.333427:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763894.333428:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763894.333428:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763894.333429:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763894.333430:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610400.
02000000:00000001:2.0:1787763894.333432:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763894.333432:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763894.333433:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bed00.
00000100:00000001:2.0:1787763894.333435:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763894.333435:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763894.333436:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763894.333443:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763894.333444:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763894.333446:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763894.333447:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763894.972247:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763894.972250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763894.972254:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763894.972255:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763894.972258:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763894.972262:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763894.972263:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763894.972264:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763894.972270:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763894.972271:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763894.972274:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763894.972275:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763894.972278:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763894.972280:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763894.972282:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763894.972357:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763894.972361:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763894.972364:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763895.356236:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763895.356243:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763895.356245:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763895.356250:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763895.356254:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763895.356257:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763895.356259:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763895.356261:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763895.356264:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763895.996246:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763895.996252:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763895.996254:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763895.996258:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763895.996259:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763895.996268:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763895.996269:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763895.996272:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763895.996275:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763895.996280:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763895.996280:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763895.996284:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763895.996285:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763895.996289:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763895.996292:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763895.996370:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763895.996374:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763895.996377:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763896.380201:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1787763896.380204:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763896.380207:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763896.380211:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763896.380213:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00100000:2.0:1787763896.380217:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763896.380218:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763896.380221:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763896.380222:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763896.380226:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763896.380227:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.380231:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763896.380233:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1787763896.380234:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21610
00000100:00000001:2.0:1787763896.380236:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763896.380244:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763896.380250:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763896.380252:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763896.380255:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763896.380258:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763896.380260:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763896.380262:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763896.380265:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763896.380266:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787763896.380269:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763896.380272:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787763896.380274:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787763896.380277:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787763896.380280:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763896.380281:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763896.380284:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763896.380285:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763896.380287:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787763896.380290:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763896.380292:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763896.380294:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763896.380297:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763896.380299:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763896.380300:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787763896.380301:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763896.380306:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bf480.
00000020:00000040:3.0:1787763896.380309:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787763896.380312:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763896.380313:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763896.380315:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763896.380317:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763896.380320:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763896.380327:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611c00.
00000100:00000001:3.0:1787763896.380332:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763896.380339:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bf480 x1874599832670592/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787763896.380346:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787763896.380349:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763896.380353:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c2965388] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bf480 x1874599832670592/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763896.380368:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763896.380370:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787763896.380373:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787763896.380374:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763896.380375:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763896.380377:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787763896.380380:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21610
00000400:00000200:3.0:1787763896.380384:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763896.380386:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763896.380388:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787763896.380390:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1787763896.380391:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763896.380395:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763896.380395:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763896.380397:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763896.380399:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1787763896.380401:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763896.380404:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763896.380405:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:3.0:1787763896.380407:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763896.380408:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:3.0:1787763896.380410:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:3.0:1787763896.380411:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:2.0:1787763896.380412:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bf480 x1874599832670592/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787763896.380415:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763896.380420:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763896.380424:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763896.380428:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763896.380432:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00000001:2.0:1787763896.380435:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:3.0:1787763896.380437:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763896.380438:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763896.380441:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763896.380444:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763896.380444:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763896.380449:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763896.380451:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763896.380452:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763896.380458:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763896.380460:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832670592:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787763896.380461:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787763896.380466:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:3.0:1787763896.380469:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1787763896.380472:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00000001:3.0:1787763896.380474:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763896.380477:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763896.380478:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763896.380482:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:3.0:1787763896.380486:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bed00.
02000000:00000010:2.0:1787763896.380491:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763896.380494:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1787763896.380497:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000200:2.0:1787763896.380503:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:3.0:1787763896.380504:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763896.380508:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763896.380510:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000100:00000001:3.0:1787763896.380510:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763896.380513:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763896.380517:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763896.380521:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
02000000:00000010:3.0:1787763896.380523:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610800.
00000100:00000200:2.0:1787763896.380531:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832670592, portal 25
00000100:00000001:3.0:1787763896.380531:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763896.380539:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bed00 x1874599832670720/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763896.380543:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763896.380547:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049859200 : -115096659692416 : ffff9751f87bf480)
00000100:00000001:3.0:1787763896.380555:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:2.0:1787763896.380561:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bf480 x1874599832670592/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763921 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763896.380561:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763896.380570:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000079751c9a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bed00 x1874599832670720/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.380578:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763896.380587:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787763896.380588:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763896.380592:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00080000:3.0:1787763896.380593:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:2.0:1787763896.380596:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832670592, offset 0
00000100:00080000:3.0:1787763896.380599:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1787763896.380603:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763896.380605:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1787763896.380609:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787763896.380610:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787763896.380616:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21610
00000400:00000200:2.0:1787763896.380621:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763896.380630:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763896.380636:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763896.380642:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763896.380646:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763896.380649:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763896.380653:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58474:58474:256:4294967295] 192.168.203.110@tcp LPNI seq info [58474:58474:8:4294967295]
00000400:00000200:2.0:1787763896.380657:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763896.380665:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000400:00000200:3.0:1787763896.380674:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:2.0:1787763896.380675:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763896.380683:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000800:00000010:2.0:1787763896.380684:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000100:00080000:3.0:1787763896.380688:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763896.380694:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:2.0:1787763896.380699:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787763896.380699:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763896.380705:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000200:2.0:1787763896.380706:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:3.0:1787763896.380709:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763896.380714:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763896.380717:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000800:00000200:2.1:1787763896.380718:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:3.0:1787763896.380721:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:2.1:1787763896.380725:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000020:00000040:3.0:1787763896.380726:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:3.0:1787763896.380731:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000100:2.0:1787763896.380737:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000020:00000001:3.0:1787763896.380737:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000010:2.0:1787763896.380744:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000100:00000040:3.0:1787763896.380746:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763896.380750:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1787763896.380751:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1787763896.380754:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763896.380760:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787763896.380761:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000020:00000040:3.0:1787763896.380765:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:2.0:1787763896.380770:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000100:00080000:3.0:1787763896.380773:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000010:2.0:1787763896.380775:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:3.0:1787763896.380780:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763896.380782:0:257575:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:3.0:1787763896.380784:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763896.380789:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:2.0:1787763896.380791:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bf480 x1874599832670592/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763921 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787763896.380793:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763896.380798:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787763896.380801:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787763896.380807:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763896.380811:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763896.380812:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000010:3.0:1787763896.380816:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc000.
00000100:00000040:2.0:1787763896.380818:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bf480 x1874599832670592/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763921 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787763896.380822:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787763896.380828:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763896.380831:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763896.380834:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763896.380835:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1787763896.380837:0:257575:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:3.0:1787763896.380839:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763896.380843:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763896.380846:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763896.380849:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612000.
00000400:00000200:2.0:1787763896.380854:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763896.380855:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763896.380858:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316284659
00000100:00100000:3.0:1787763896.380865:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc000 x1874599832670848/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:2.0:1787763896.380867:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00000001:3.0:1787763896.380875:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787763896.380880:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763896.380886:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000040:3.0:1787763896.380886:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3fc1c6c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc000 x1874599832670848/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.380898:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316284659
00000100:00000001:3.0:1787763896.380899:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763896.380904:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7ba0.
00000100:00000040:3.0:1787763896.380905:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1787763896.380912:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1787763896.380913:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00080000:3.0:1787763896.380918:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763896.380920:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:3.0:1787763896.380926:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787763896.380927:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763896.380930:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787763896.380932:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00080000:3.0:1787763896.380936:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1787763896.380941:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763896/real 1787763896]  req@ffff9751f87bf480 x1874599832670592/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763896.380968:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763896.380970:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763896.380972:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763896.380976:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763896.380978:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:2.0:1787763896.380981:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763896.380986:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bf480 x1874599832670592/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763896.380995:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bf480 x1874599832670592/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.381002:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763896.381003:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.381006:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763896.381008:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763896.381012:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bf480 x1874599832670592/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.381019:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763896.381024:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bf480 x1874599832670592/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.381033:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763896.381036:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.381038:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763896.381040:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.381042:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763896.381044:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763896.381046:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763896.381049:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763896.381050:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763896.381054:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bf480 x1874599832670592/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763896.381075:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832670592:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787763896.381081:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.381083:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763896.381091:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bf480 x1874599832670592/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.381107:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763896.381109:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763896.381112:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763896.381115:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763896.381117:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763896.381119:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787763896.381121:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763896.381124:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611c00.
02000000:00000001:2.0:1787763896.381128:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763896.381130:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763896.381132:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bf480.
00000100:00000001:2.0:1787763896.381155:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763896.381156:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.381159:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.381165:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763896.381167:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763896.381169:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763896.381171:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763896.381173:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763896.381177:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bed00 x1874599832670720/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.381182:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763896.381183:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763896.381185:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763896.381186:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763896.381188:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763896.381189:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832670720:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763896.381192:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763896.381194:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763896.381196:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763896.381198:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763896.381202:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763896.381204:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763896.381208:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763896.381211:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000400:00000010:2.0:1787763896.381215:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000200:2.0:1787763896.381219:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832670720, portal 10
00000100:00000001:2.0:1787763896.381221:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763896.381222:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049857280 : -115096659694336 : ffff9751f87bed00)
00000100:00000040:2.0:1787763896.381228:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bed00 x1874599832670720/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763921 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.381235:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763896.381237:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763896.381239:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000200:2.0:1787763896.381241:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832670720, offset 0
00000400:00000200:2.0:1787763896.381246:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763896.381250:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763896.381254:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763896.381258:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763896.381261:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763896.381263:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763896.381265:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763896.381269:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58475:58475:256:4294967295] 192.168.203.110@tcp LPNI seq info [58475:58475:8:4294967295]
00000400:00000200:2.0:1787763896.381274:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763896.381282:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763896.381286:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763896.381289:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000800:00000200:2.0:1787763896.381295:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763896.381298:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787763896.381303:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787763896.381306:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:2.1:1787763896.381306:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000100:00080000:3.0:1787763896.381314:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21610
00000800:00000100:2.0:1787763896.381316:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763896.381321:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000400:00000200:3.0:1787763896.381324:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763896.381328:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763896.381333:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1787763896.381338:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:3.0:1787763896.381340:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1787763896.381346:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000100:00000040:3.0:1787763896.381347:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000010:2.0:1787763896.381351:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:3.0:1787763896.381353:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763896.381356:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:3.0:1787763896.381358:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763896.381363:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000200:2.0:1787763896.381365:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bed00 x1874599832670720/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763921 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787763896.381368:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763896.381372:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:3.0:1787763896.381376:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763896.381380:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000001:2.0:1787763896.381382:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000001:3.0:1787763896.381384:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:2.0:1787763896.381389:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bed00 x1874599832670720/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763921 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787763896.381390:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:3.0:1787763896.381399:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763896.381405:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763896.381407:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763896.381410:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763896.381411:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763896.381414:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763896.381418:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1787763896.381420:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:3.0:1787763896.381426:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000040:2.0:1787763896.381427:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000100:00000001:3.0:1787763896.381431:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763896.381436:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:3.0:1787763896.381438:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787763896.381441:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316283424
02000000:00000001:3.0:1787763896.381444:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763896.381446:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316284659
02000000:00000001:3.0:1787763896.381450:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763896.381452:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316283424
00000100:00000001:3.0:1787763896.381454:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787763896.381456:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316284659
00000008:00100000:3.0:1787763896.381460:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:2.0:1787763896.381461:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000008:00000001:3.0:1787763896.381467:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763896.381468:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763896.381472:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763896.381473:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000010:3.0:1787763896.381480:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87be1c0.
00000100:00100000:2.0:1787763896.381482:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763896/real 1787763896]  req@ffff9751f87bed00 x1874599832670720/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787763896.381486:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:3.0:1787763896.381492:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763896.381495:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763896.381500:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000001:2.0:1787763896.381502:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:3.0:1787763896.381503:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000001:2.0:1787763896.381507:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:3.0:1787763896.381508:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763896.381512:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
02000000:00000010:3.0:1787763896.381516:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610c00.
00000400:00000200:2.0:1787763896.381518:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763896.381522:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:3.0:1787763896.381523:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763896.381526:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763896.381534:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bed00 x1874599832670720/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763896.381534:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87be1c0 x1874599832670976/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787763896.381562:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:2.0:1787763896.381567:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bed00 x1874599832670720/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763896.381568:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763896.381582:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763896.381585:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.381587:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763896.381588:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763896.381592:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bed00 x1874599832670720/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.381598:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763896.381602:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bed00 x1874599832670720/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.381608:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763896.381610:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.381612:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763896.381613:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.381615:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763896.381629:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763896.381633:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763896.381636:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763896.381639:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763896.381643:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bed00 x1874599832670720/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763896.381651:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832670720:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763896.381655:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763896.381659:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc000 x1874599832670848/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.381667:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763896.381670:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763896.381676:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:3.0:1787763896.381677:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e2011c6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87be1c0 x1874599832670976/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787763896.381680:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763896.381687:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763896.381696:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832670848:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763896.381697:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763896.381705:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:3.0:1787763896.381706:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21621)
02000000:00000001:2.0:1787763896.381709:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763896.381713:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763896.381722:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763896.381727:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763896.381730:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763896.381734:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763896.381736:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000400:00000010:2.0:1787763896.381740:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000200:2.0:1787763896.381744:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832670848, portal 10
00000100:00000001:2.0:1787763896.381746:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763896.381747:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049845760 : -115096659705856 : ffff9751f87bc000)
00000100:00000040:2.0:1787763896.381754:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc000 x1874599832670848/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763921 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.381761:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763896.381764:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763896.381766:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000200:2.0:1787763896.381769:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832670848, offset 0
00000400:00000200:2.0:1787763896.381773:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763896.381777:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763896.381781:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763896.381784:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763896.381787:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763896.381790:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763896.381792:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763896.381795:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58476:58476:256:4294967295] 192.168.203.110@tcp LPNI seq info [58476:58476:8:4294967295]
00000400:00000200:2.0:1787763896.381800:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763896.381807:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763896.381812:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763896.381816:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000800:00000200:2.0:1787763896.381821:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763896.381824:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787763896.381829:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763896.381831:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787763896.381835:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763896.381837:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000400:00000200:2.0:1787763896.381840:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763896.381844:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763896.381847:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:2.0:1787763896.381848:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:2.0:1787763896.381850:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763896.381854:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc000 x1874599832670848/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763921 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.381862:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763896.381866:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc000 x1874599832670848/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763921 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.381873:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763896.381875:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763896.381879:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763896.381882:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787763896.381885:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763896.381887:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316283412
00000100:00000001:2.0:1787763896.381890:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316284659
00000100:00000001:2.0:1787763896.381892:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316283412
00000100:00000001:2.0:1787763896.381894:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316284659
00000100:00000001:2.0:1787763896.381896:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763896.381899:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763896.381901:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763896.381905:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763896/real 1787763896]  req@ffff9751f87bc000 x1874599832670848/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763896.381913:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763896.381915:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763896.381916:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763896.381919:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763896.381921:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:2.0:1787763896.381923:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763896.381926:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc000 x1874599832670848/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763896.381935:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc000 x1874599832670848/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.381958:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763896.381961:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.381963:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763896.381965:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763896.381968:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc000 x1874599832670848/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.381973:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763896.381976:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc000 x1874599832670848/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.381980:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763896.381982:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.381983:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763896.381984:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.381985:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763896.381986:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763896.381987:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763896.381990:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763896.381991:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763896.381994:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc000 x1874599832670848/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763896.381999:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832670848:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763896.382002:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.382003:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763896.382005:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bed00 x1874599832670720/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.382009:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763896.382010:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763896.382012:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763896.382013:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763896.382014:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763896.382015:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763896.382017:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763896.382018:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610800.
02000000:00000001:2.0:1787763896.382020:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763896.382021:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763896.382023:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bed00.
00000100:00000001:2.0:1787763896.382025:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763896.382025:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.382026:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763896.382028:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc000 x1874599832670848/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.382033:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763896.382033:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763896.382034:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763896.382036:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763896.382037:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763896.382038:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763896.382040:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763896.382041:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612000.
02000000:00000001:2.0:1787763896.382043:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763896.382074:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763896.382076:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc000.
00000100:00000001:2.0:1787763896.382079:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763896.382081:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.382083:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.382089:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763896.382091:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763896.382093:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763896.382094:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763896.382096:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763896.382100:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87be1c0 x1874599832670976/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.382108:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763896.382109:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763896.382111:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763896.382113:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763896.382115:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763896.382118:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832670976:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763896.382121:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763896.382123:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763896.382124:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763896.382126:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763896.382128:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763896.382129:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763896.382132:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763896.382153:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000400:00000010:2.0:1787763896.382156:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000200:2.0:1787763896.382158:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832670976, portal 4
00000100:00000001:2.0:1787763896.382160:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763896.382161:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049854400 : -115096659697216 : ffff9751f87be1c0)
00000100:00000040:2.0:1787763896.382169:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87be1c0 x1874599832670976/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763921 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.382173:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763896.382175:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763896.382177:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000200:2.0:1787763896.382179:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832670976, offset 0
00000400:00000200:2.0:1787763896.382182:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763896.382185:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763896.382188:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763896.382190:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763896.382192:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763896.382194:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763896.382195:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763896.382198:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58477:58477:256:4294967295] 192.168.203.110@tcp LPNI seq info [58477:58477:8:4294967295]
00000400:00000200:2.0:1787763896.382200:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763896.382205:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763896.382207:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763896.382209:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000800:00000200:2.0:1787763896.382212:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763896.382214:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787763896.382217:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763896.382219:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787763896.382221:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763896.382223:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000400:00000200:2.0:1787763896.382225:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763896.382227:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763896.382229:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:2.0:1787763896.382230:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:2.0:1787763896.382232:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763896.382235:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87be1c0 x1874599832670976/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763921 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.382240:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763896.382242:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87be1c0 x1874599832670976/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763921 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.382247:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763896.382248:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763896.382250:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763896.382252:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787763896.382254:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763896.382256:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316283430
00000100:00000001:2.0:1787763896.382258:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316284659
00000100:00000001:2.0:1787763896.382259:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316283430
00000100:00000001:2.0:1787763896.382261:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316284659
00000100:00000001:2.0:1787763896.382262:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763896.382263:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763896.382264:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763896.382267:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763896/real 1787763896]  req@ffff9751f87be1c0 x1874599832670976/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763896.382271:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763896.382272:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763896.382274:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763896.382275:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763896.382276:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:2.0:1787763896.382277:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763896.382279:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87be1c0 x1874599832670976/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763896.382284:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87be1c0 x1874599832670976/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.382288:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763896.382289:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.382291:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763896.382292:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763896.382294:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87be1c0 x1874599832670976/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.382298:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763896.382300:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87be1c0 x1874599832670976/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.382305:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763896.382307:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.382308:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763896.382309:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.382310:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763896.382311:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763896.382312:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763896.382315:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763896.382316:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763896.382319:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87be1c0 x1874599832670976/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763896.382323:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832670976:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763896.382325:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.382327:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763896.382328:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87be1c0 x1874599832670976/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763921 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763896.382332:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763896.382333:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763896.382334:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763896.382336:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763896.382336:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763896.382337:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787763896.382338:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763896.382339:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610c00.
02000000:00000001:2.0:1787763896.382341:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763896.382342:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763896.382343:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87be1c0.
00000100:00000001:2.0:1787763896.382345:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763896.382346:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.382347:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763896.382349:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763896.382350:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763896.382352:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763896.382353:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763897.020212:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763897.020220:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763897.020222:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763897.020226:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763897.020230:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763897.020235:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763897.020238:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763897.020240:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763897.020242:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763897.020264:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763897.020269:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763897.020273:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763897.020278:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763897.020280:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763897.020283:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763897.020286:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763897.020288:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763897.020290:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763897.020292:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763897.020295:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763897.020296:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763897.020358:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763897.020363:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763897.020365:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763897.404255:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763897.404263:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763897.404265:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763897.404271:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763897.404276:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763897.404280:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763897.404283:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763897.404287:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763897.404290:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763897.660280:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316284659, at 4316284672
00000100:00000010:3.0:1787763897.660302:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7ba0.
00000100:00000040:2.0:1787763897.660316:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316284659, at 4316284672
00000100:00000010:2.0:1787763897.660335:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00010000:00000001:3.0:1787763898.044232:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763898.044233:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763898.044238:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763898.044241:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763898.044244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763898.044246:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763898.044248:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763898.044253:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763898.044254:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763898.044258:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763898.044260:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763898.044266:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763898.044270:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763898.044273:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763898.044276:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763898.044326:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763898.044331:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763898.044334:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763898.428226:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763898.428235:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763898.428238:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763898.428244:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763898.428248:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763898.428252:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763898.428255:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763898.428257:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763898.428260:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763899.068282:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763899.068287:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763899.068290:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763899.068294:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763899.068295:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763899.068304:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763899.068304:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763899.068309:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763899.068314:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763899.068317:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763899.068319:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763899.068320:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763899.068325:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763899.068328:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763899.068331:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763899.068376:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763899.068380:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763899.068383:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763899.452246:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763899.452256:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763899.452259:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763899.452266:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763899.452270:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763899.452274:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763899.452277:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763899.452280:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763899.452283:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763900.092232:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763900.092234:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763900.092240:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763900.092242:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763900.092246:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763900.092250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763900.092252:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763900.092254:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763900.092257:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763900.092261:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763900.092263:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763900.092264:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763900.092270:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763900.092273:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763900.092276:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763900.092325:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763900.092329:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763900.092332:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763900.476211:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763900.476219:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763900.476222:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763900.476228:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763900.476233:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763900.476236:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763900.476239:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763900.476241:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763900.476245:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763901.116207:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763901.116213:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763901.116216:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763901.116217:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763901.116220:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763901.116224:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763901.116227:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763901.116227:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763901.116234:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763901.116236:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763901.116243:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763901.116251:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.116255:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763901.116258:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:1.0:1787763901.116262:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000100:00000001:2.0:1787763901.116262:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763901.116267:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763901.116270:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763901.500234:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763901.500242:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763901.500245:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763901.500252:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:0.0:1787763901.500256:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763901.500257:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763901.500262:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787763901.500263:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787763901.500266:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763901.500267:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763901.500271:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763901.500271:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763901.500278:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763901.500278:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763901.500285:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21616
00000400:00000200:0.0:1787763901.500292:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763901.500298:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763901.500301:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763901.500303:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763901.500305:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763901.500307:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763901.500309:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763901.500312:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763901.500313:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763901.500315:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763901.500318:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763901.500320:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763901.500321:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763901.500323:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763901.500325:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763901.500326:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763901.500328:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763901.500330:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763901.500332:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763901.500333:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763901.500335:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763901.500337:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763901.500339:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763901.500341:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763901.500342:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763901.500349:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763901.500353:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763901.500356:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763901.500357:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763901.500359:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763901.500361:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763901.500365:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763901.500371:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763901.500376:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763901.500383:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832671104/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763901.500391:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763901.500394:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763901.500398:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832671104/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763901.500415:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763901.500417:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763901.500420:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763901.500421:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763901.500422:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763901.500424:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763901.500425:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21616
00000400:00000200:0.0:1787763901.500428:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763901.500430:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763901.500432:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763901.500434:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763901.500436:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763901.500438:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763901.500440:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763901.500440:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1787763901.500445:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763901.500446:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00000001:2.0:1787763901.500446:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1787763901.500447:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1787763901.500450:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1787763901.500451:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763901.500455:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763901.500456:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:2.0:1787763901.500461:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832671104/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763901.500462:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763901.500465:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763901.500468:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763901.500471:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763901.500476:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763901.500479:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763901.500486:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763901.500487:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763901.500490:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763901.500491:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763901.500497:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763901.500499:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763901.500500:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763901.500504:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1787763901.500504:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:0.0:1787763901.500509:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763901.500509:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763901.500513:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763901.500515:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763901.500517:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787763901.500521:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000100:00100000:2.0:1787763901.500523:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832671104:192.168.203.110@tcp:250:kworker.0
02000000:00000001:0.0:1787763901.500524:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763901.500527:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763901.500530:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763901.500532:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787763901.500532:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763901.500537:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763901.500538:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000010:0.0:1787763901.500540:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
02000000:00000001:2.0:1787763901.500543:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763901.500547:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763901.500553:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
00000100:00000001:0.0:1787763901.500554:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763901.500557:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763901.500562:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832671232/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787763901.500568:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763901.500575:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:2.0:1787763901.500575:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000100:00000001:0.0:1787763901.500577:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763901.500583:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832671232/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1787763901.500585:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000200:2.0:1787763901.500594:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832671104, portal 25
00000100:00000001:2.0:1787763901.500599:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763901.500601:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763901.500603:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000100:00080000:0.0:1787763901.500605:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763901.500609:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763901.500611:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763901.500614:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:2.0:1787763901.500615:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832671104/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763926 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763901.500616:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763901.500622:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21616
00000400:00000200:0.0:1787763901.500628:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763901.500633:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763901.500638:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787763901.500639:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763901.500642:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000040:2.0:1787763901.500646:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763901.500647:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763901.500652:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000010:2.0:1787763901.500652:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000001:0.0:1787763901.500657:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000200:2.0:1787763901.500658:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832671104, offset 0
00000020:00000001:0.0:1787763901.500659:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763901.500662:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763901.500664:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763901.500666:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763901.500669:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:2.0:1787763901.500670:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000020:00000001:0.0:1787763901.500671:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763901.500677:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763901.500678:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763901.500681:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787763901.500683:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763901.500684:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763901.500688:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763901.500690:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763901.500693:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763901.500694:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
02000000:00000001:0.0:1787763901.500695:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763901.500699:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1787763901.500703:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:0.0:1787763901.500704:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763901.500707:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763901.500708:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:2.0:1787763901.500710:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000008:00000001:0.0:1787763901.500711:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763901.500715:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763901.500718:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000010:0.0:1787763901.500721:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000400:00000200:2.0:1787763901.500723:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000020:00000040:0.0:1787763901.500725:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763901.500730:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:2.0:1787763901.500731:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58478:58478:256:4294967295] 192.168.203.110@tcp LPNI seq info [58478:58478:8:4294967295]
02000000:00000001:0.0:1787763901.500732:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763901.500736:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763901.500740:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763901.500743:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763901.500743:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000010:0.0:1787763901.500749:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763901.500752:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763901.500760:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832671360/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787763901.500762:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1787763901.500774:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763901.500778:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787763901.500778:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787763901.500785:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832671360/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:2.0:1787763901.500787:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000100:00000001:0.0:1787763901.500793:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763901.500796:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:2.0:1787763901.500801:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763901.500802:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763901.500807:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:2.0:1787763901.500809:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000100:00080000:0.0:1787763901.500810:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763901.500813:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763901.500815:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763901.500817:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:2.1:1787763901.500821:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00080000:0.0:1787763901.500823:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21616
00000800:00000200:2.1:1787763901.500827:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000400:00000200:0.0:1787763901.500828:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763901.500834:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763901.500838:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000100:2.0:1787763901.500839:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787763901.500844:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000010:2.0:1787763901.500844:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000100:00000001:0.0:1787763901.500849:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763901.500851:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763901.500852:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787763901.500853:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:0.0:1787763901.500854:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763901.500857:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763901.500860:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763901.500863:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000400:00000200:2.0:1787763901.500864:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:0.0:1787763901.500865:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763901.500870:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000200:2.0:1787763901.500872:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000100:00000040:0.0:1787763901.500873:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763901.500877:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1787763901.500877:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000040:0.0:1787763901.500881:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763901.500883:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763901.500884:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000020:00000040:0.0:1787763901.500885:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763901.500889:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763901.500892:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763901.500893:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e1c0 x1874599832671104/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763926 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763901.500894:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763901.500897:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763901.500901:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763901.500906:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763901.500921:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763901.500925:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763901.500929:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763901.500932:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000010:0.0:1787763901.500933:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763901.500937:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000040:2.0:1787763901.500939:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832671104/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763926 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763901.500940:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763901.500943:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763901.500946:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763901.500948:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763901.500951:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763901.500956:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:2.0:1787763901.500958:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763901.500960:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763901.500961:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00100000:0.0:1787763901.500967:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832671488/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763901.500969:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763901.500977:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1787763901.500978:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763901.500981:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763901.500983:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316289779
00000100:00000040:0.0:1787763901.500988:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832671488/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1787763901.500992:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00000001:2.0:1787763901.501001:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763901.501002:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763901.501009:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21626)
00000100:00000001:2.0:1787763901.501010:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316289779
00000100:00000010:2.0:1787763901.501016:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7900.
00000100:00000001:2.0:1787763901.501020:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787763901.501025:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763901.501028:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763901.501031:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763901.501037:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763901/real 1787763901]  req@ffff9751f876e1c0 x1874599832671104/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763901.501047:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763901.501049:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763901.501051:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763901.501054:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763901.501055:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:2.0:1787763901.501058:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763901.501062:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832671104/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763901.501077:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832671104/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.501085:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763901.501087:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.501089:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763901.501091:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763901.501095:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832671104/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.501104:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763901.501109:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832671104/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.501117:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763901.501120:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.501121:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763901.501123:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.501125:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763901.501128:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763901.501131:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763901.501152:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763901.501155:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763901.501159:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832671104/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763901.501169:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832671104:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787763901.501174:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.501176:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763901.501180:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832671104/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.501195:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763901.501197:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763901.501200:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763901.501203:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763901.501205:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763901.501207:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787763901.501209:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763901.501212:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014800.
02000000:00000001:2.0:1787763901.501216:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763901.501218:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763901.501219:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:2.0:1787763901.501224:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763901.501225:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.501227:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.501234:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763901.501235:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763901.501238:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763901.501239:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763901.501241:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763901.501245:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832671232/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.501253:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763901.501254:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763901.501257:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763901.501259:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763901.501261:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763901.501264:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832671232:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763901.501268:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763901.501270:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763901.501271:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763901.501273:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763901.501277:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763901.501279:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763901.501283:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763901.501285:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000400:00000010:2.0:1787763901.501289:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000200:2.0:1787763901.501292:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832671232, portal 10
00000100:00000001:2.0:1787763901.501294:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763901.501296:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00000040:2.0:1787763901.501303:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832671232/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763926 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.501310:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763901.501312:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763901.501314:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000200:2.0:1787763901.501317:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832671232, offset 0
00000400:00000200:2.0:1787763901.501321:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763901.501325:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763901.501329:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763901.501333:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763901.501336:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763901.501338:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763901.501341:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763901.501344:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58479:58479:256:4294967295] 192.168.203.110@tcp LPNI seq info [58479:58479:8:4294967295]
00000400:00000200:2.0:1787763901.501349:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763901.501355:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763901.501360:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763901.501364:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000800:00000200:2.0:1787763901.501368:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763901.501371:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787763901.501376:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763901.501379:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1787763901.501383:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763901.501385:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000400:00000200:2.0:1787763901.501388:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763901.501392:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763901.501395:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:2.0:1787763901.501397:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:2.0:1787763901.501400:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763901.501404:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e580 x1874599832671232/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763926 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.501413:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763901.501416:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832671232/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763926 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.501423:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763901.501425:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763901.501428:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763901.501431:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787763901.501435:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763901.501437:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316288544
00000100:00000001:2.0:1787763901.501440:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316289779
00000100:00000001:2.0:1787763901.501443:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316288544
00000100:00000001:2.0:1787763901.501445:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316289779
00000100:00000001:2.0:1787763901.501448:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763901.501450:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763901.501452:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763901.501456:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763901/real 1787763901]  req@ffff9751f876e580 x1874599832671232/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763901.501464:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763901.501465:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763901.501467:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763901.501470:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763901.501472:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:2.0:1787763901.501474:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763901.501477:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832671232/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763901.501486:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832671232/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.501500:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763901.501502:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.501504:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763901.501505:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763901.501509:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832671232/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.501516:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763901.501521:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832671232/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.501528:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763901.501530:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.501532:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763901.501533:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.501536:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763901.501548:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763901.501551:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763901.501555:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763901.501557:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763901.501562:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832671232/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763901.501570:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832671232:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763901.501573:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763901.501576:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f0c0 x1874599832671360/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.501584:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763901.501586:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763901.501588:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763901.501590:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763901.501592:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763901.501595:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832671360:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763901.501599:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763901.501601:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763901.501602:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763901.501604:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763901.501607:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763901.501609:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763901.501612:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763901.501615:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000400:00000010:2.0:1787763901.501618:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000200:2.0:1787763901.501621:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832671360, portal 10
00000100:00000001:2.0:1787763901.501623:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763901.501624:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049530560 : -115096660021056 : ffff9751f876f0c0)
00000100:00000040:2.0:1787763901.501630:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f0c0 x1874599832671360/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763926 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.501638:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763901.501640:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763901.501642:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000200:2.0:1787763901.501644:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832671360, offset 0
00000400:00000200:2.0:1787763901.501648:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763901.501652:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763901.501655:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763901.501658:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763901.501661:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763901.501663:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763901.501666:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763901.501670:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58480:58480:256:4294967295] 192.168.203.110@tcp LPNI seq info [58480:58480:8:4294967295]
00000400:00000200:2.0:1787763901.501674:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763901.501680:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763901.501686:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763901.501689:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000800:00000200:2.0:1787763901.501693:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763901.501695:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787763901.501700:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763901.501703:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1787763901.501707:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763901.501710:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000400:00000200:2.0:1787763901.501713:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763901.501717:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763901.501720:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:2.0:1787763901.501721:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:2.0:1787763901.501724:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763901.501727:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f0c0 x1874599832671360/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763926 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.501736:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763901.501739:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f0c0 x1874599832671360/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763926 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.501746:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763901.501748:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763901.501750:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763901.501753:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787763901.501756:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763901.501758:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316288532
00000100:00000001:2.0:1787763901.501760:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316289779
00000100:00000001:2.0:1787763901.501763:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316288532
00000100:00000001:2.0:1787763901.501765:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316289779
00000100:00000001:2.0:1787763901.501767:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763901.501769:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763901.501771:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763901.501775:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763901/real 1787763901]  req@ffff9751f876f0c0 x1874599832671360/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763901.501784:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763901.501785:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763901.501787:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763901.501789:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763901.501791:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:2.0:1787763901.501793:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763901.501796:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f0c0 x1874599832671360/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763901.501806:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f0c0 x1874599832671360/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.501814:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763901.501815:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.501817:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763901.501819:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763901.501822:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f0c0 x1874599832671360/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.501830:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763901.501834:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f0c0 x1874599832671360/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.501842:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763901.501845:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.501846:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763901.501848:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.501850:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763901.501851:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763901.501854:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763901.501857:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763901.501860:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763901.501864:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f0c0 x1874599832671360/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763901.501874:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832671360:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763901.501878:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763901.501881:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832671488/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.501888:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763901.501890:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763901.501892:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763901.501894:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763901.501896:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763901.501899:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832671488:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763901.501903:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763901.501904:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763901.501906:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763901.501922:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763901.501926:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763901.501928:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763901.501932:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763901.501934:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000400:00000010:2.0:1787763901.501937:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000200:2.0:1787763901.501941:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832671488, portal 4
00000100:00000001:2.0:1787763901.501942:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763901.501944:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
00000100:00000040:2.0:1787763901.501950:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832671488/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763926 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.501957:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763901.501959:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763901.501962:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000200:2.0:1787763901.501964:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832671488, offset 0
00000400:00000200:2.0:1787763901.501969:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763901.501972:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763901.501976:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763901.501979:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763901.501982:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763901.501985:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763901.501987:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763901.501990:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58481:58481:256:4294967295] 192.168.203.110@tcp LPNI seq info [58481:58481:8:4294967295]
00000400:00000200:2.0:1787763901.501994:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763901.502002:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763901.502007:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763901.502009:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000800:00000200:2.0:1787763901.502013:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763901.502016:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787763901.502020:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763901.502023:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1787763901.502027:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763901.502029:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000400:00000200:2.0:1787763901.502032:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763901.502036:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763901.502039:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:2.0:1787763901.502041:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:2.0:1787763901.502043:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763901.502047:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876ed00 x1874599832671488/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763926 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.502056:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763901.502059:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832671488/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763926 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.502066:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763901.502068:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763901.502070:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763901.502073:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787763901.502076:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763901.502079:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316288549
00000100:00000001:2.0:1787763901.502081:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316289779
00000100:00000001:2.0:1787763901.502083:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316288549
00000100:00000001:2.0:1787763901.502086:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316289779
00000100:00000001:2.0:1787763901.502088:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763901.502090:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763901.502092:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763901.502096:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763901/real 1787763901]  req@ffff9751f876ed00 x1874599832671488/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763901.502104:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763901.502105:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763901.502107:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763901.502109:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763901.502111:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:2.0:1787763901.502113:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763901.502116:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832671488/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763901.502126:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832671488/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.502150:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763901.502152:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.502154:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763901.502155:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763901.502160:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832671488/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.502167:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763901.502171:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832671488/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.502180:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763901.502182:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.502184:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763901.502185:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.502187:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763901.502189:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763901.502192:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763901.502195:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763901.502198:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763901.502202:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832671488/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763901.502211:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832671488:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763901.502214:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.502216:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763901.502219:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832671232/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.502226:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763901.502228:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763901.502230:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763901.502233:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763901.502234:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763901.502236:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763901.502238:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763901.502240:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:2.0:1787763901.502243:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763901.502244:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763901.502246:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:2.0:1787763901.502248:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763901.502250:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.502251:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763901.502255:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f0c0 x1874599832671360/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.502269:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763901.502270:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763901.502272:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763901.502274:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763901.502276:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763901.502277:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763901.502279:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763901.502281:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015c00.
02000000:00000001:2.0:1787763901.502283:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763901.502284:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763901.502285:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f0c0.
00000100:00000001:2.0:1787763901.502288:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763901.502289:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.502290:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763901.502294:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832671488/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763926 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763901.502300:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763901.502301:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763901.502302:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763901.502305:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763901.502306:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763901.502307:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787763901.502309:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763901.502310:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016800.
02000000:00000001:2.0:1787763901.502313:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763901.502314:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763901.502316:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:2.0:1787763901.502318:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763901.502319:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.502321:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763901.502324:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763901.502326:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763901.502328:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763901.502330:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763902.140210:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763902.140218:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763902.140220:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763902.140225:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763902.140225:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763902.140230:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763902.140231:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763902.140235:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763902.140236:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763902.140239:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763902.140240:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763902.140243:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763902.140245:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763902.140250:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763902.140251:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763902.140254:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763902.140258:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763902.140260:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763902.140262:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763902.140263:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763902.140265:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763902.140321:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763902.140325:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763902.140328:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763902.524207:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763902.524216:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763902.524219:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763902.524224:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763902.524229:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763902.524232:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763902.524234:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763902.524237:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763902.524240:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763902.780201:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316289779, at 4316289792
00000100:00000010:3.0:1787763902.780215:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7900.
00000100:00000040:2.0:1787763902.780253:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316289779, at 4316289792
00000100:00000010:2.0:1787763902.780262:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00000100:00000001:2.0:1787763903.164238:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763903.164240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763903.164245:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763903.164245:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763903.164249:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763903.164251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763903.164253:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763903.164254:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763903.164260:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763903.164261:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763903.164263:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763903.164264:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763903.164268:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763903.164271:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763903.164273:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763903.164356:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763903.164360:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763903.164362:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763903.548206:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763903.548214:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763903.548217:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763903.548223:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763903.548226:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763903.548230:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763903.548233:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763903.548236:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763903.548239:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763904.188222:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763904.188225:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763904.188229:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763904.188231:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763904.188232:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763904.188234:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763904.188238:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763904.188239:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763904.188243:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787763904.188244:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763904.188246:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763904.188248:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763904.188250:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763904.188252:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763904.188253:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763904.188255:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763904.188258:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763904.188260:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763904.188261:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763904.188266:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763904.188268:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763904.188310:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763904.188314:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763904.188318:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763904.572249:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763904.572257:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763904.572260:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763904.572266:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763904.572271:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763904.572275:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763904.572278:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763904.572281:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763904.572284:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763905.212261:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763905.212263:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763905.212271:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763905.212272:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763905.212277:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763905.212280:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763905.212282:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763905.212290:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763905.212291:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763905.212292:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763905.212300:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763905.212305:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763905.212309:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763905.212312:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763905.212323:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763905.212387:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763905.212392:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763905.212396:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763905.596241:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763905.596250:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763905.596252:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763905.596257:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763905.596261:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763905.596264:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763905.596267:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763905.596270:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763905.596273:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763906.236224:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763906.236224:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763906.236233:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763906.236234:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763906.236238:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763906.236242:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763906.236242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763906.236247:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763906.236252:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.236256:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763906.236260:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.236264:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763906.236311:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763906.236316:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763906.236326:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763906.236473:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763906.236479:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763906.236483:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763906.620264:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763906.620273:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763906.620276:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763906.620283:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763906.620288:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763906.620294:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787763906.620297:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763906.620298:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763906.620304:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.620313:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763906.620314:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763906.620320:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763906.620325:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763906.620331:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763906.620338:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21621
00000400:00000200:0.0:1787763906.620348:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763906.620356:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763906.620360:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763906.620363:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763906.620367:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763906.620370:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763906.620373:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763906.620377:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763906.620380:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763906.620383:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763906.620386:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763906.620389:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763906.620392:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763906.620396:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763906.620398:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763906.620400:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763906.620403:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763906.620406:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763906.620410:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763906.620413:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763906.620416:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763906.620420:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763906.620423:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763906.620425:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763906.620428:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763906.620437:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000020:00000040:0.0:1787763906.620441:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763906.620445:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763906.620447:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763906.620450:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763906.620453:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763906.620457:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763906.620466:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763906.620473:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763906.620482:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832671616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763906.620493:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763906.620497:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763906.620502:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832671616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763906.620526:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763906.620530:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763906.620534:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763906.620535:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763906.620538:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763906.620542:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763906.620548:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21621
00000400:00000200:0.0:1787763906.620553:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763906.620554:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:0.0:1787763906.620560:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787763906.620562:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763906.620566:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00080000:0.0:1787763906.620567:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763906.620574:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000040:2.0:1787763906.620579:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cf00 x1874599832671616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763906.620581:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763906.620592:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763906.620602:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763906.620604:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763906.620605:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763906.620614:0:304425:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:2.0:1787763906.620615:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:0.0:1787763906.620620:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
02000000:00000001:2.0:1787763906.620622:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:0.0:1787763906.620629:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:2.0:1787763906.620629:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787763906.620640:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00100000:2.0:1787763906.620641:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832671616:192.168.203.110@tcp:250:kworker.0
00000020:00000001:0.0:1787763906.620649:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787763906.620650:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763906.620656:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000001:0.0:1787763906.620657:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:2.0:1787763906.620660:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763906.620663:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:0.0:1787763906.620667:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763906.620672:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000010:2.0:1787763906.620674:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
00000100:00000040:0.0:1787763906.620684:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:2.0:1787763906.620684:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763906.620694:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787763906.620701:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:0.0:1787763906.620706:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000400:00000010:2.0:1787763906.620708:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000100:00080000:0.0:1787763906.620720:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000010:2.0:1787763906.620722:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000200:2.0:1787763906.620732:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832671616, portal 25
00000100:00000001:0.0:1787763906.620734:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763906.620735:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763906.620738:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049521920 : -115096660029696 : ffff9751f876cf00)
02000000:00000001:0.0:1787763906.620742:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:2.0:1787763906.620749:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cf00 x1874599832671616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763906.620751:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763906.620764:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763906.620767:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1787763906.620770:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:2.0:1787763906.620776:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763906.620780:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000008:00100000:0.0:1787763906.620781:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000200:2.0:1787763906.620786:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832671616, offset 0
00000008:00000001:0.0:1787763906.620790:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763906.620796:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763906.620797:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763906.620809:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000400:00000200:2.0:1787763906.620815:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:0.0:1787763906.620825:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000200:2.0:1787763906.620825:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
02000000:00000001:0.0:1787763906.620837:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:2.0:1787763906.620839:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:0.0:1787763906.620846:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763906.620849:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:0.0:1787763906.620854:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:2.0:1787763906.620856:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
02000000:00000001:0.0:1787763906.620858:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1787763906.620862:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
02000000:00000001:0.0:1787763906.620866:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763906.620867:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58482:58482:256:4294967295] 192.168.203.110@tcp LPNI seq info [58482:58482:8:4294967295]
00000400:00000200:2.0:1787763906.620889:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000010:0.0:1787763906.620890:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763906.620897:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763906.620908:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00100000:0.0:1787763906.620916:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832671744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.0:1787763906.620921:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763906.620930:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000100:00000001:0.0:1787763906.620939:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:2.0:1787763906.620943:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763906.620946:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787763906.620949:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787763906.620959:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763906.620960:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832671744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.1:1787763906.620969:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1787763906.620980:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763906.620986:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000400:00000200:2.0:1787763906.620998:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1787763906.620999:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763906.621006:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787763906.621009:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763906.621017:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:2.0:1787763906.621019:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00080000:0.0:1787763906.621025:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1787763906.621028:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1787763906.621029:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763906.621034:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763906.621042:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:2.0:1787763906.621043:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cf00 x1874599832671616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787763906.621048:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21621
00000400:00000200:0.0:1787763906.621060:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763906.621071:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763906.621075:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787763906.621076:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763906.621083:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cf00 x1874599832671616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763931 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763906.621087:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763906.621097:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763906.621104:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763906.621109:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763906.621110:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763906.621116:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763906.621118:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763906.621126:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:2.0:1787763906.621131:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763906.621160:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316294900
00000100:00000010:2.0:1787763906.621168:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000020:00000040:0.0:1787763906.621173:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000100:00000001:2.0:1787763906.621175:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000020:00000001:0.0:1787763906.621180:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763906.621185:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000100:00000001:2.0:1787763906.621187:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316294900
00000020:00000001:0.0:1787763906.621189:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000010:2.0:1787763906.621191:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7d80.
00000020:00000001:0.0:1787763906.621198:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000001:2.0:1787763906.621198:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787763906.621206:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763906.621209:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763906.621213:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763906.621214:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:0.0:1787763906.621222:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:2.0:1787763906.621222:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763906/real 1787763906]  req@ffff9751f876cf00 x1874599832671616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763906.621234:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000001:2.0:1787763906.621239:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:0.0:1787763906.621241:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000001:2.0:1787763906.621242:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763906.621248:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000020:00000040:0.0:1787763906.621249:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:2.0:1787763906.621253:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763906.621257:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00080000:0.0:1787763906.621258:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763906.621261:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:0.0:1787763906.621265:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763906.621268:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cf00 x1874599832671616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763906.621272:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763906.621282:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:2.0:1787763906.621283:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cf00 x1874599832671616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763906.621297:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763906.621304:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787763906.621306:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787763906.621307:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000008:00100000:0.0:1787763906.621313:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1787763906.621313:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763906.621317:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000008:00000001:0.0:1787763906.621323:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763906.621326:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cf00 x1874599832671616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763906.621329:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763906.621338:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:0.0:1787763906.621343:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
00000100:00000040:2.0:1787763906.621344:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cf00 x1874599832671616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787763906.621353:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000001:2.0:1787763906.621357:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:0.0:1787763906.621361:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1787763906.621361:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787763906.621367:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763906.621367:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763906.621369:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763906.621379:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1787763906.621379:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
02000000:00000001:0.0:1787763906.621384:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787763906.621384:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:0.0:1787763906.621388:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763906.621393:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000010:0.0:1787763906.621396:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00080000:2.0:1787763906.621402:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787763906.621403:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763906.621408:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00100000:0.0:1787763906.621415:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832671872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:2.0:1787763906.621420:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cf00 x1874599832671616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763906.621452:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763906.621457:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763906.621458:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832671616:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787763906.621472:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.621477:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763906.621478:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832671872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787763906.621482:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cf00 x1874599832671616/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.621538:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763906.621546:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000001:0.0:1787763906.621547:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763906.621550:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
00000100:00000040:0.0:1787763906.621555:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:2.0:1787763906.621555:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763906.621561:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763906.621566:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000100:00000040:0.0:1787763906.621567:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000020:00000001:2.0:1787763906.621571:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
00000100:00080000:0.0:1787763906.621575:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000010:2.0:1787763906.621575:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016800.
00000100:00080000:0.0:1787763906.621587:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
02000000:00000001:2.0:1787763906.621588:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763906.621592:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000001:0.0:1787763906.621594:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000010:2.0:1787763906.621596:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cf00.
00000100:00080000:0.0:1787763906.621598:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763906.621602:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787763906.621604:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763906.621605:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.621611:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787763906.621617:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21621
00000100:00000001:2.0:1787763906.621622:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000400:00000200:0.0:1787763906.621623:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763906.621625:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763906.621634:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763906.621636:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000400:00000200:0.0:1787763906.621639:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787763906.621639:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763906.621647:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e940 x1874599832671744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787763906.621648:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763906.621661:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763906.621664:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763906.621669:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763906.621671:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1787763906.621672:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763906.621676:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763906.621683:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763906.621684:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00100000:2.0:1787763906.621693:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832671744:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787763906.621694:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763906.621705:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:0.0:1787763906.621706:0:304425:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:2.0:1787763906.621708:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:0.0:1787763906.621710:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
02000000:00000001:2.0:1787763906.621711:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763906.621718:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:2.0:1787763906.621719:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763906.621724:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
00000020:00000040:0.0:1787763906.621725:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
02000000:00000001:2.0:1787763906.621728:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763906.621732:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000200:2.0:1787763906.621738:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:0.0:1787763906.621743:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000010:2.0:1787763906.621743:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000400:00000010:2.0:1787763906.621754:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000040:0.0:1787763906.621755:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000200:2.0:1787763906.621759:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832671744, portal 10
00000100:00000001:0.0:1787763906.621760:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763906.621762:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763906.621767:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049528640 : -115096660022976 : ffff9751f876e940)
00000100:00000040:0.0:1787763906.621771:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763906.621776:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:2.0:1787763906.621778:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e940 x1874599832671744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787763906.621786:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:2.0:1787763906.621790:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:0.0:1787763906.621800:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000040:2.0:1787763906.621800:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763906.621805:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000001:0.0:1787763906.621807:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763906.621810:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832671744, offset 0
02000000:00000001:0.0:1787763906.621813:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1787763906.621817:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
02000000:00000001:0.0:1787763906.621821:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1787763906.621824:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787763906.621832:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763906.621836:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:0.0:1787763906.621842:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:2.0:1787763906.621844:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000008:00100000:0.0:1787763906.621849:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000400:00000200:2.0:1787763906.621850:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763906.621856:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763906.621863:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000008:00000001:0.0:1787763906.621865:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763906.621869:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58483:58483:256:4294967295] 192.168.203.110@tcp LPNI seq info [58483:58483:8:4294967295]
00000100:00000001:0.0:1787763906.621888:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763906.621894:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000010:0.0:1787763906.621899:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000400:00000200:2.0:1787763906.621905:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000020:00000040:0.0:1787763906.621907:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000800:00000200:2.0:1787763906.621913:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
02000000:00000001:0.0:1787763906.621918:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000010:2.0:1787763906.621919:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
02000000:00000001:0.0:1787763906.621922:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787763906.621929:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763906.621931:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:2.0:1787763906.621935:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
02000000:00000001:0.0:1787763906.621936:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763906.621946:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787763906.621947:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763906.621955:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
02000000:00000010:0.0:1787763906.621959:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000800:00000100:2.0:1787763906.621962:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763906.621969:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000100:00000001:0.0:1787763906.621971:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763906.621973:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763906.621985:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00100000:0.0:1787763906.621988:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832672000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787763906.621988:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:2.0:1787763906.621993:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:2.0:1787763906.621998:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763906.622004:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e940 x1874599832671744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763906.622029:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763906.622034:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1787763906.622035:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763906.622042:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e940 x1874599832671744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763931 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763906.622048:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832672000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622104:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763906.622106:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763906.622107:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763906.622111:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763906.622117:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21631)
00000100:00000040:2.0:1787763906.622117:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787763906.622127:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763906.622130:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316293664
00000100:00000001:2.0:1787763906.622149:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316294900
00000100:00000001:2.0:1787763906.622152:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316293665
00000100:00000001:2.0:1787763906.622154:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316294900
00000100:00000001:2.0:1787763906.622156:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763906.622157:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763906.622160:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763906.622166:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763906/real 1787763906]  req@ffff9751f876e940 x1874599832671744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763906.622172:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763906.622174:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763906.622175:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763906.622177:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763906.622178:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:2.0:1787763906.622180:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763906.622182:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e940 x1874599832671744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763906.622188:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e940 x1874599832671744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622192:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763906.622193:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.622194:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763906.622195:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763906.622197:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e940 x1874599832671744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622203:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763906.622205:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e940 x1874599832671744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622210:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763906.622211:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.622212:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763906.622213:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.622214:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763906.622225:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763906.622226:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763906.622228:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763906.622230:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763906.622232:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e940 x1874599832671744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763906.622237:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832671744:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763906.622240:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763906.622243:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f840 x1874599832671872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622248:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763906.622249:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763906.622250:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763906.622252:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763906.622253:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763906.622255:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832671872:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763906.622257:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763906.622258:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763906.622260:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763906.622261:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763906.622264:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763906.622266:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763906.622268:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763906.622270:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000400:00000010:2.0:1787763906.622273:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000200:2.0:1787763906.622275:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832671872, portal 10
00000100:00000001:2.0:1787763906.622276:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763906.622277:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049532480 : -115096660019136 : ffff9751f876f840)
00000100:00000040:2.0:1787763906.622282:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f840 x1874599832671872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622286:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763906.622288:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763906.622290:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000200:2.0:1787763906.622291:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832671872, offset 0
00000400:00000200:2.0:1787763906.622294:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763906.622297:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763906.622299:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763906.622301:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763906.622303:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763906.622305:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763906.622307:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763906.622309:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58484:58484:256:4294967295] 192.168.203.110@tcp LPNI seq info [58484:58484:8:4294967295]
00000400:00000200:2.0:1787763906.622312:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763906.622317:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763906.622320:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763906.622322:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000800:00000200:2.0:1787763906.622325:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763906.622327:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787763906.622331:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763906.622333:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1787763906.622336:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763906.622338:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000400:00000200:2.0:1787763906.622340:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763906.622343:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763906.622345:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:2.0:1787763906.622346:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:2.0:1787763906.622347:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763906.622350:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f840 x1874599832671872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622355:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763906.622357:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f840 x1874599832671872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763931 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622362:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763906.622363:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763906.622364:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763906.622366:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787763906.622368:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763906.622369:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316293653
00000100:00000001:2.0:1787763906.622371:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316294900
00000100:00000001:2.0:1787763906.622372:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316293653
00000100:00000001:2.0:1787763906.622374:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316294900
00000100:00000001:2.0:1787763906.622376:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763906.622377:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763906.622378:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763906.622381:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763906/real 1787763906]  req@ffff9751f876f840 x1874599832671872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763906.622388:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763906.622390:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763906.622392:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763906.622394:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763906.622395:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:2.0:1787763906.622398:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763906.622402:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f840 x1874599832671872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763906.622411:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f840 x1874599832671872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622418:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763906.622420:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.622422:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763906.622424:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763906.622428:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f840 x1874599832671872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622436:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763906.622439:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f840 x1874599832671872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622444:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763906.622446:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.622447:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763906.622448:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.622449:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763906.622450:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763906.622452:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763906.622454:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763906.622455:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763906.622458:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f840 x1874599832671872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763906.622462:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832671872:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763906.622465:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.622467:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763906.622469:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e940 x1874599832671744/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622473:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763906.622475:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763906.622476:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763906.622478:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763906.622479:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763906.622480:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763906.622482:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763906.622483:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015c00.
02000000:00000001:2.0:1787763906.622485:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763906.622486:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763906.622487:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e940.
00000100:00000001:2.0:1787763906.622491:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763906.622492:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.622493:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763906.622495:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f840 x1874599832671872/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622499:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763906.622500:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763906.622501:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763906.622503:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763906.622504:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763906.622504:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763906.622505:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763906.622506:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:2.0:1787763906.622508:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763906.622508:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763906.622509:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f840.
00000100:00000001:2.0:1787763906.622510:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763906.622511:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.622512:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.622515:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763906.622516:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763906.622517:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763906.622519:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763906.622520:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763906.622523:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cf00 x1874599832672000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622528:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763906.622529:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763906.622530:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763906.622532:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763906.622533:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763906.622535:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832672000:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763906.622537:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763906.622539:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763906.622540:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763906.622541:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763906.622543:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763906.622544:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763906.622547:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763906.622549:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000400:00000010:2.0:1787763906.622552:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000200:2.0:1787763906.622554:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832672000, portal 4
00000100:00000001:2.0:1787763906.622556:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763906.622557:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049521920 : -115096660029696 : ffff9751f876cf00)
00000100:00000040:2.0:1787763906.622561:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cf00 x1874599832672000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622566:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763906.622567:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763906.622569:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000200:2.0:1787763906.622570:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832672000, offset 0
00000400:00000200:2.0:1787763906.622573:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763906.622575:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763906.622578:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763906.622580:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763906.622582:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763906.622584:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763906.622586:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763906.622588:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58485:58485:256:4294967295] 192.168.203.110@tcp LPNI seq info [58485:58485:8:4294967295]
00000400:00000200:2.0:1787763906.622591:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763906.622595:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763906.622598:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763906.622600:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000800:00000200:2.0:1787763906.622603:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763906.622605:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787763906.622608:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763906.622610:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1787763906.622613:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763906.622614:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000400:00000200:2.0:1787763906.622616:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763906.622618:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763906.622620:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:2.0:1787763906.622621:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:2.0:1787763906.622623:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763906.622625:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cf00 x1874599832672000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622630:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763906.622632:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cf00 x1874599832672000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763931 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622637:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763906.622638:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763906.622640:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763906.622642:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787763906.622644:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763906.622645:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316293670
00000100:00000001:2.0:1787763906.622647:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316294900
00000100:00000001:2.0:1787763906.622649:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316293670
00000100:00000001:2.0:1787763906.622652:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316294900
00000100:00000001:2.0:1787763906.622654:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763906.622655:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763906.622657:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763906.622659:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763906/real 1787763906]  req@ffff9751f876cf00 x1874599832672000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763906.622663:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763906.622664:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763906.622665:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763906.622666:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763906.622667:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:2.0:1787763906.622668:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763906.622670:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cf00 x1874599832672000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763906.622676:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cf00 x1874599832672000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622680:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763906.622681:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.622682:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763906.622683:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763906.622685:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cf00 x1874599832672000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622689:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763906.622691:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cf00 x1874599832672000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622695:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763906.622697:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.622697:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763906.622698:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.622700:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763906.622701:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763906.622703:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763906.622706:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763906.622707:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763906.622710:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cf00 x1874599832672000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763906.622714:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832672000:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763906.622716:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.622717:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763906.622719:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cf00 x1874599832672000/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763931 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763906.622723:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763906.622724:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763906.622725:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763906.622726:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763906.622727:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763906.622728:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787763906.622730:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763906.622731:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014800.
02000000:00000001:2.0:1787763906.622732:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763906.622733:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763906.622734:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cf00.
00000100:00000001:2.0:1787763906.622735:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763906.622736:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.622737:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763906.622739:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763906.622740:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763906.622741:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763906.622742:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763907.260279:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763907.260287:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763907.260288:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763907.260296:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763907.260299:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763907.260302:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763907.260303:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763907.260306:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763907.260311:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763907.260313:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763907.260314:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763907.260321:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763907.260322:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763907.260328:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763907.260329:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763907.260331:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763907.260333:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763907.260337:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763907.260338:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763907.260347:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763907.260347:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763907.260397:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787763907.260400:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787763907.260403:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763907.644304:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763907.644317:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763907.644320:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763907.644328:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763907.644334:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763907.644339:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763907.644342:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763907.644346:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763907.644350:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763907.900255:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316294900, at 4316294912
00000100:00000010:3.0:1787763907.900274:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7d80.
00000100:00000040:2.0:1787763907.900290:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316294900, at 4316294912
00000100:00000010:2.0:1787763907.900299:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00000100:00000001:2.0:1787763908.284237:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763908.284237:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763908.284244:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763908.284247:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763908.284251:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763908.284252:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763908.284255:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763908.284259:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763908.284260:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763908.284262:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763908.284268:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763908.284273:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763908.284277:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763908.284281:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763908.284285:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763908.284317:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763908.284320:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763908.284324:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763908.668277:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763908.668287:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763908.668290:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763908.668297:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763908.668302:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763908.668306:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763908.668310:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763908.668315:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763908.668319:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763909.308216:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763909.308223:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763909.308227:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763909.308229:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763909.308231:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763909.308238:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763909.308238:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763909.308241:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763909.308245:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763909.308248:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763909.308251:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763909.308251:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763909.308257:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763909.308262:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763909.308266:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763909.308308:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763909.308314:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763909.308319:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763909.692224:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763909.692233:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763909.692234:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763909.692240:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763909.692244:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763909.692247:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763909.692249:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763909.692252:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763909.692255:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763910.332207:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763910.332213:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763910.332218:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763910.332220:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763910.332223:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763910.332223:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763910.332241:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763910.332250:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763910.332254:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763910.332261:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763910.332266:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763910.332271:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763910.332274:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763910.332277:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763910.332280:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763910.332313:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763910.332317:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763910.332319:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763910.716298:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763910.716306:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763910.716309:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763910.716314:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763910.716319:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763910.716323:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763910.716326:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763910.716329:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763910.716332:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763911.356233:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763911.356234:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763911.356239:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763911.356242:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763911.356244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763911.356246:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763911.356247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763911.356254:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763911.356255:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763911.356257:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763911.356261:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787763911.356262:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763911.356264:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763911.356267:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787763911.356268:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763911.356269:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763911.356274:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763911.356274:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763911.356276:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763911.356279:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763911.356285:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763911.356335:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763911.356338:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763911.356340:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787763911.740232:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763911.740242:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787763911.740243:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763911.740245:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1787763911.740249:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:1.0:1787763911.740251:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763911.740255:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763911.740257:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00100000:3.0:1787763911.740262:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:1.0:1787763911.740267:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21626
00000100:00000001:3.0:1787763911.740269:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763911.740275:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:1.0:1787763911.740278:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763911.740280:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763911.740285:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:1.0:1787763911.740289:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787763911.740292:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763911.740294:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763911.740300:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763911.740305:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763911.740308:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763911.740311:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763911.740315:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763911.740317:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787763911.740320:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763911.740324:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787763911.740327:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787763911.740339:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787763911.740343:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763911.740345:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763911.740348:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763911.740351:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763911.740354:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787763911.740358:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763911.740361:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763911.740365:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763911.740368:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763911.740372:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763911.740375:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763911.740378:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763911.740384:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932940.
00000020:00000040:1.0:1787763911.740388:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787763911.740392:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763911.740393:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763911.740396:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763911.740399:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763911.740402:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763911.740409:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb400.
00000100:00000001:1.0:1787763911.740415:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763911.740432:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932940 x1874599832672128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763911.740452:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763911.740457:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763911.740464:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000af90eaa] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932940 x1874599832672128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763911.740483:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763911.740487:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763911.740491:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787763911.740493:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763911.740495:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763911.740498:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763911.740502:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21626
00000400:00000200:1.0:1787763911.740507:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763911.740512:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763911.740515:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763911.740519:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763911.740522:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763911.740524:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763911.740527:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763911.740529:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763911.740531:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787763911.740533:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763911.740535:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787763911.740537:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787763911.740539:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787763911.740543:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763911.740544:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763911.740546:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763911.740547:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763911.740549:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787763911.740551:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763911.740553:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763911.740555:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763911.740560:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763911.740562:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763911.740563:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763911.740568:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763911.740571:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763911.740572:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763911.740575:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763911.740580:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930780.
00000020:00000040:1.0:1787763911.740583:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763911.740585:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763911.740587:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763911.740589:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763911.740591:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763911.740594:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763911.740598:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9000.
00000100:00000001:1.0:1787763911.740603:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763911.740621:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930780 x1874599832672256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763911.740629:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763911.740631:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763911.740634:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3977950] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930780 x1874599832672256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763911.740640:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763911.740642:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763911.740644:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787763911.740646:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763911.740647:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763911.740649:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763911.740651:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21626
00000400:00000200:1.0:1787763911.740654:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763911.740657:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763911.740658:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763911.740661:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763911.740662:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763911.740664:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763911.740665:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763911.740667:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763911.740668:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787763911.740669:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763911.740671:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787763911.740672:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787763911.740674:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787763911.740676:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763911.740677:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763911.740678:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763911.740680:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763911.740684:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00000001:3.0:1787763911.740686:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:1.0:1787763911.740687:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763911.740690:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1787763911.740691:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763911.740695:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763911.740698:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1787763911.740700:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932940 x1874599832672128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787763911.740701:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763911.740706:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763911.740709:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763911.740714:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763911.740715:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763911.740717:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763911.740718:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763911.740722:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932580.
02000000:00000001:3.0:1787763911.740723:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:1.0:1787763911.740727:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787763911.740728:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763911.740732:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763911.740735:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00100000:3.0:1787763911.740737:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832672128:192.168.203.110@tcp:250:kworker.0
02000000:00000001:1.0:1787763911.740739:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763911.740743:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763911.740745:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763911.740748:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763911.740753:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8000.
00000100:00000001:1.0:1787763911.740757:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763911.740764:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932580 x1874599832672384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763911.740772:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763911.740774:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763911.740779:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d026924] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932580 x1874599832672384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763911.740789:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763911.740792:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787763911.740793:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:1.0:1787763911.740795:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
02000000:00000001:3.0:1787763911.740799:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:1.0:1787763911.740800:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:3.0:1787763911.740803:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763911.740804:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787763911.740808:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
02000000:00000001:3.0:1787763911.740809:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:1.0:1787763911.740810:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763911.740814:0:304425:0:(import.c:485:import_select_connection()) Process entered
02000000:00000010:3.0:1787763911.740815:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610c00.
00000100:00080000:1.0:1787763911.740817:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21626
02000000:00000001:3.0:1787763911.740819:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763911.740823:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763911.740829:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1787763911.740830:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:1.0:1787763911.740832:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000010:3.0:1787763911.740835:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000100:00000040:1.0:1787763911.740847:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763911.740850:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763911.740854:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763911.740856:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763911.740858:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763911.740860:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000400:00000010:3.0:1787763911.740860:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000020:00000001:1.0:1787763911.740864:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763911.740867:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787763911.740869:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000200:3.0:1787763911.740869:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832672128, portal 25
00000020:00000001:1.0:1787763911.740871:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:3.0:1787763911.740875:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000040:1.0:1787763911.740876:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763911.740877:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497731904 : -115097211819712 : ffff9751d7932940)
00000100:00000001:1.0:1787763911.740880:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763911.740883:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763911.740887:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:3.0:1787763911.740887:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932940 x1874599832672128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787763911.740890:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:1.0:1787763911.740895:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763911.740900:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763911.740904:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787763911.740906:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:1.0:1787763911.740908:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763911.740913:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763911.740914:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787763911.740916:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:3.0:1787763911.740918:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000008:00100000:1.0:1787763911.740920:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000200:3.0:1787763911.740923:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832672128, offset 0
00000008:00000001:1.0:1787763911.740924:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763911.740928:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763911.740933:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933840.
00000400:00000200:3.0:1787763911.740934:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000020:00000040:1.0:1787763911.740937:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787763911.740940:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763911.740941:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763911.740942:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1787763911.740944:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:1.0:1787763911.740945:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763911.740948:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763911.740953:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca800.
00000400:00000200:3.0:1787763911.740954:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:1.0:1787763911.740956:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763911.740960:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:1.0:1787763911.740964:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933840 x1874599832672512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787763911.740968:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763911.740974:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1787763911.740976:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1787763911.740978:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763911.740982:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58486:58486:256:4294967295] 192.168.203.110@tcp LPNI seq info [58486:58486:8:4294967295]
00000100:00000001:1.0:1787763911.740986:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763911.740991:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:1.0:1787763911.740995:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000083dd6d32] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933840 x1874599832672512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787763911.741007:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:1.0:1787763911.741011:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787763911.741016:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000040:1.0:1787763911.741017:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21636)
00000800:00000010:3.0:1787763911.741023:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763911.741032:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763911.741037:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787763911.741044:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763911.741048:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1787763911.741053:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763911.741055:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763911.741059:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763911.741064:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763911.741069:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763911.741072:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763911.741077:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763911.741080:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932940 x1874599832672128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741088:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763911.741091:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932940 x1874599832672128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763936 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741096:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763911.741097:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763911.741102:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763911.741105:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763911.741107:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316300019
00000100:00000010:3.0:1787763911.741111:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d404e0.
00000100:00000001:3.0:1787763911.741116:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787763911.741121:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316300019
00000100:00000010:3.0:1787763911.741123:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d409c0.
00000100:00000001:3.0:1787763911.741125:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787763911.741127:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763911.741129:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763911.741131:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763911.741156:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763911/real 1787763911]  req@ffff9751d7932940 x1874599832672128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763911.741164:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763911.741166:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763911.741168:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763911.741171:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763911.741172:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763911.741174:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763911.741177:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932940 x1874599832672128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763911.741183:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932940 x1874599832672128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741192:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763911.741194:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763911.741195:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763911.741197:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763911.741199:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932940 x1874599832672128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741204:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763911.741207:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932940 x1874599832672128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741212:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763911.741214:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763911.741215:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763911.741216:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763911.741218:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763911.741219:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763911.741221:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763911.741223:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763911.741225:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763911.741227:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932940 x1874599832672128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763911.741232:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832672128:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787763911.741236:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763911.741238:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930780 x1874599832672256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741242:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763911.741243:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763911.741245:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763911.741247:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763911.741250:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763911.741253:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832672256:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763911.741257:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763911.741259:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763911.741260:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763911.741263:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763911.741266:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787763911.741268:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763911.741272:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763911.741275:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763911.741278:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763911.741282:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832672256, portal 10
00000100:00000001:3.0:1787763911.741284:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763911.741285:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497723264 : -115097211828352 : ffff9751d7930780)
00000100:00000040:3.0:1787763911.741293:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930780 x1874599832672256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741300:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763911.741303:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763911.741306:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763911.741308:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832672256, offset 0
00000400:00000200:3.0:1787763911.741313:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763911.741318:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763911.741322:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763911.741324:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763911.741326:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763911.741328:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763911.741329:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763911.741331:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58487:58487:256:4294967295] 192.168.203.110@tcp LPNI seq info [58487:58487:8:4294967295]
00000400:00000200:3.0:1787763911.741334:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763911.741339:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763911.741342:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763911.741345:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763911.741348:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763911.741350:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787763911.741353:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763911.741355:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1787763911.741358:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763911.741359:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763911.741361:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763911.741364:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763911.741366:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763911.741367:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763911.741369:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763911.741372:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930780 x1874599832672256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741377:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763911.741379:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930780 x1874599832672256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763936 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741384:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763911.741385:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763911.741388:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763911.741392:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787763911.741395:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763911.741398:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316298784
00000100:00000001:3.0:1787763911.741400:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316300019
00000100:00000001:3.0:1787763911.741402:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316298784
00000100:00000001:3.0:1787763911.741405:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316300019
00000100:00000001:3.0:1787763911.741407:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763911.741409:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763911.741411:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763911.741416:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763911/real 1787763911]  req@ffff9751d7930780 x1874599832672256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763911.741433:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763911.741434:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763911.741436:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763911.741439:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763911.741440:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763911.741442:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763911.741445:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930780 x1874599832672256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763911.741450:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930780 x1874599832672256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741455:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763911.741456:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763911.741457:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763911.741458:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763911.741460:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930780 x1874599832672256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741465:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763911.741467:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930780 x1874599832672256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741472:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763911.741474:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763911.741475:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763911.741475:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763911.741476:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763911.741486:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763911.741488:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763911.741490:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763911.741491:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763911.741494:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930780 x1874599832672256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763911.741499:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832672256:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763911.741502:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763911.741503:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763911.741505:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932940 x1874599832672128/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741509:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763911.741511:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763911.741513:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610c00.
02000000:00000001:3.0:1787763911.741514:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763911.741515:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763911.741516:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787763911.741518:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763911.741520:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb400.
02000000:00000001:3.0:1787763911.741522:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763911.741524:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763911.741525:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932940.
00000100:00000001:3.0:1787763911.741527:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763911.741528:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763911.741529:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763911.741531:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930780 x1874599832672256/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741535:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763911.741536:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763911.741537:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787763911.741538:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763911.741539:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763911.741540:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763911.741541:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763911.741542:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9000.
02000000:00000001:3.0:1787763911.741543:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763911.741544:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763911.741545:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930780.
00000100:00000001:3.0:1787763911.741547:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763911.741548:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763911.741549:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763911.741554:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763911.741555:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763911.741557:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763911.741558:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763911.741559:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763911.741561:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932580 x1874599832672384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741566:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763911.741567:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763911.741568:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763911.741570:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763911.741571:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763911.741573:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832672384:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763911.741575:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763911.741576:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763911.741577:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763911.741579:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763911.741581:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787763911.741582:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763911.741585:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763911.741586:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763911.741589:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763911.741591:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832672384, portal 10
00000100:00000001:3.0:1787763911.741592:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763911.741593:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497730944 : -115097211820672 : ffff9751d7932580)
00000100:00000040:3.0:1787763911.741597:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932580 x1874599832672384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741602:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763911.741603:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763911.741605:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763911.741606:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832672384, offset 0
00000400:00000200:3.0:1787763911.741609:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763911.741612:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763911.741614:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763911.741617:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763911.741619:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763911.741620:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763911.741622:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763911.741624:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58488:58488:256:4294967295] 192.168.203.110@tcp LPNI seq info [58488:58488:8:4294967295]
00000400:00000200:3.0:1787763911.741627:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763911.741631:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763911.741635:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763911.741637:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763911.741639:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763911.741641:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787763911.741644:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763911.741646:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1787763911.741649:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763911.741650:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763911.741652:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763911.741654:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763911.741656:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763911.741658:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763911.741659:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763911.741661:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932580 x1874599832672384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741667:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763911.741668:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932580 x1874599832672384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763936 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741673:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763911.741674:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763911.741676:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763911.741678:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787763911.741680:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763911.741681:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316298772
00000100:00000001:3.0:1787763911.741683:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316300019
00000100:00000001:3.0:1787763911.741685:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316298772
00000100:00000001:3.0:1787763911.741686:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316300019
00000100:00000001:3.0:1787763911.741687:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763911.741688:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763911.741690:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763911.741692:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763911/real 1787763911]  req@ffff9751d7932580 x1874599832672384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763911.741696:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763911.741697:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763911.741698:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763911.741700:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763911.741701:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763911.741702:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763911.741704:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932580 x1874599832672384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763911.741709:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932580 x1874599832672384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741713:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763911.741714:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763911.741715:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763911.741716:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763911.741718:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932580 x1874599832672384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741722:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763911.741725:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932580 x1874599832672384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741729:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763911.741731:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763911.741732:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763911.741733:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763911.741734:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763911.741735:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763911.741737:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763911.741739:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763911.741740:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763911.741742:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932580 x1874599832672384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763911.741747:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832672384:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763911.741749:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763911.741751:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933840 x1874599832672512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741755:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763911.741756:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763911.741757:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763911.741758:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763911.741759:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763911.741761:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832672512:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763911.741763:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763911.741764:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763911.741765:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763911.741766:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763911.741768:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610c00.
02000000:00000001:3.0:1787763911.741769:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763911.741771:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763911.741773:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763911.741775:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763911.741778:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832672512, portal 4
00000100:00000001:3.0:1787763911.741780:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763911.741782:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497735744 : -115097211815872 : ffff9751d7933840)
00000100:00000040:3.0:1787763911.741787:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933840 x1874599832672512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741794:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763911.741796:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763911.741798:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763911.741801:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832672512, offset 0
00000400:00000200:3.0:1787763911.741805:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763911.741809:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763911.741814:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763911.741816:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763911.741819:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763911.741822:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763911.741824:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763911.741828:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58489:58489:256:4294967295] 192.168.203.110@tcp LPNI seq info [58489:58489:8:4294967295]
00000400:00000200:3.0:1787763911.741832:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763911.741851:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763911.741856:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763911.741860:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763911.741864:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763911.741867:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787763911.741872:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763911.741875:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1787763911.741879:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763911.741881:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763911.741884:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763911.741888:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763911.741892:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763911.741893:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763911.741895:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763911.741898:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933840 x1874599832672512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741904:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763911.741906:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933840 x1874599832672512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763936 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741910:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763911.741911:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763911.741913:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763911.741916:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787763911.741918:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763911.741919:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316298789
00000100:00000001:3.0:1787763911.741921:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316300019
00000100:00000001:3.0:1787763911.741923:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316298789
00000100:00000001:3.0:1787763911.741924:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316300019
00000100:00000001:3.0:1787763911.741926:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763911.741927:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763911.741928:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763911.741930:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763911/real 1787763911]  req@ffff9751d7933840 x1874599832672512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763911.741936:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763911.741937:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763911.741938:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763911.741940:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763911.741941:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763911.741942:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763911.741944:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933840 x1874599832672512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763911.741949:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933840 x1874599832672512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741954:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763911.741955:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763911.741956:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763911.741957:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763911.741960:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933840 x1874599832672512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741964:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763911.741966:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933840 x1874599832672512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741970:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763911.741971:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763911.741972:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763911.741973:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763911.741974:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763911.741975:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763911.741977:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763911.741979:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763911.741980:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763911.741982:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933840 x1874599832672512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763911.741987:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832672512:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763911.741989:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763911.741990:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763911.741992:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932580 x1874599832672384/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.741997:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763911.741998:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763911.742000:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787763911.742001:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763911.742002:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763911.742003:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763911.742005:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763911.742006:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8000.
02000000:00000001:3.0:1787763911.742007:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763911.742008:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763911.742009:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932580.
00000100:00000001:3.0:1787763911.742011:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763911.742012:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763911.742013:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763911.742015:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933840 x1874599832672512/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763936 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763911.742019:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763911.742020:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763911.742022:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610c00.
02000000:00000001:3.0:1787763911.742024:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763911.742025:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763911.742027:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787763911.742029:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763911.742031:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca800.
02000000:00000001:3.0:1787763911.742033:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763911.742034:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763911.742035:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933840.
00000100:00000001:3.0:1787763911.742039:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763911.742040:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763911.742042:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763911.742045:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763911.742047:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763911.742049:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763911.742050:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763912.380214:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763912.380222:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763912.380224:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763912.380228:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763912.380232:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763912.380234:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763912.380237:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763912.380239:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763912.380242:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763912.380287:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763912.380290:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763912.380294:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763912.380295:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763912.380297:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763912.380298:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763912.380344:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763912.380348:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763912.380351:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763912.764245:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763912.764252:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763912.764253:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763912.764257:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763912.764261:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763912.764264:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763912.764267:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763912.764269:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763912.764271:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763913.020217:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316300019, at 4316300032
00000100:00000010:3.0:1787763913.020231:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d409c0.
00000100:00000040:2.0:1787763913.020261:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316300019, at 4316300032
00000100:00000010:2.0:1787763913.020272:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d404e0.
00000100:00000001:2.0:1787763913.404232:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763913.404234:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763913.404240:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763913.404241:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763913.404245:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763913.404245:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763913.404249:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763913.404251:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763913.404253:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787763913.404256:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763913.404256:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763913.404260:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763913.404261:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763913.404263:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763913.404264:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763913.404267:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763913.404268:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763913.404270:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763913.404271:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763913.404276:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763913.404278:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763913.404325:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763913.404328:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763913.404330:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763913.788209:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763913.788218:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763913.788220:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763913.788226:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763913.788230:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763913.788234:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763913.788237:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763913.788240:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763913.788243:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763914.428206:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763914.428208:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763914.428211:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763914.428216:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763914.428217:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763914.428220:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763914.428220:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763914.428227:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763914.428228:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763914.428228:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763914.428234:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763914.428238:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763914.428241:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763914.428244:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763914.428247:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763914.428271:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763914.428275:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763914.428278:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763914.812234:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763914.812240:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763914.812242:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763914.812247:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763914.812250:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763914.812254:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763914.812256:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763914.812259:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763914.812261:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763915.452219:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763915.452225:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763915.452226:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763915.452231:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763915.452234:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763915.452234:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763915.452237:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763915.452240:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763915.452241:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763915.452247:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763915.452252:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763915.452257:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763915.452260:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763915.452263:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763915.452267:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763915.452272:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763915.452276:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763915.452279:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763915.836217:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763915.836224:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763915.836228:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763915.836233:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763915.836237:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763915.836240:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763915.836243:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763915.836245:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763915.836248:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763916.476208:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763916.476208:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763916.476214:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763916.476215:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763916.476219:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763916.476221:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763916.476223:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763916.476226:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763916.476228:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763916.476229:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787763916.476232:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763916.476233:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763916.476237:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763916.476238:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787763916.476240:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763916.476241:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763916.476246:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763916.476247:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763916.476252:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763916.476256:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763916.476257:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763916.476315:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763916.476321:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763916.476324:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763916.860234:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763916.860242:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763916.860244:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763916.860248:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763916.860248:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763916.860255:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763916.860255:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763916.860259:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763916.860264:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21631
00000100:00100000:2.0:1787763916.860265:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763916.860270:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763916.860273:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1787763916.860274:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763916.860277:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763916.860280:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1787763916.860282:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787763916.860283:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763916.860286:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763916.860291:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763916.860293:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763916.860295:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763916.860298:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763916.860301:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763916.860302:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763916.860304:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763916.860306:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763916.860308:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763916.860309:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763916.860312:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763916.860313:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763916.860315:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763916.860316:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763916.860318:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763916.860321:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763916.860322:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763916.860325:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763916.860328:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763916.860329:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763916.860331:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763916.860333:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763916.860341:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763916.860344:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763916.860347:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763916.860355:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763916.860357:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763916.860359:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763916.860363:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763916.860369:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787763916.860374:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763916.860382:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832672640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763916.860389:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763916.860392:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763916.860396:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832672640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763916.860411:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763916.860413:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763916.860416:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763916.860417:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763916.860422:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763916.860425:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763916.860428:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21631
00000400:00000200:0.0:1787763916.860431:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763916.860433:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763916.860436:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787763916.860436:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1787763916.860441:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763916.860443:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763916.860444:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787763916.860446:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763916.860448:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763916.860451:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763916.860454:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763916.860456:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00000040:3.0:1787763916.860456:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832672640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787763916.860459:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763916.860463:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763916.860467:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763916.860472:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:3.0:1787763916.860474:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:0.0:1787763916.860477:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763916.860477:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763916.860482:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1787763916.860482:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:0.0:1787763916.860487:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:3.0:1787763916.860487:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763916.860488:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:3.0:1787763916.860491:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787763916.860493:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00100000:3.0:1787763916.860498:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832672640:192.168.203.110@tcp:250:kworker.0
00000100:00080000:0.0:1787763916.860499:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763916.860504:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763916.860508:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1787763916.860509:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763916.860511:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763916.860512:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763916.860516:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1787763916.860519:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000010:3.0:1787763916.860522:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610c00.
02000000:00000001:0.0:1787763916.860523:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763916.860527:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1787763916.860527:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1787763916.860533:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:3.0:1787763916.860538:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000008:00000001:0.0:1787763916.860539:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763916.860542:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787763916.860543:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000100:00000010:0.0:1787763916.860547:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763916.860551:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000010:3.0:1787763916.860551:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
02000000:00000001:0.0:1787763916.860558:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763916.860559:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763916.860560:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763916.860561:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:3.0:1787763916.860562:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832672640, portal 25
02000000:00000001:0.0:1787763916.860563:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763916.860568:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763916.860570:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
02000000:00000010:0.0:1787763916.860571:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000040:3.0:1787763916.860577:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832672640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763941 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763916.860578:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763916.860588:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00100000:0.0:1787763916.860592:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832672768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:3.0:1787763916.860594:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763916.860597:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763916.860602:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832672640, offset 0
00000400:00000200:3.0:1787763916.860611:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763916.860612:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1787763916.860619:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763916.860623:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763916.860626:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000040:0.0:1787763916.860632:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832672768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787763916.860633:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763916.860641:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763916.860647:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763916.860653:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:0.0:1787763916.860659:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763916.860660:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58490:58490:256:4294967295] 192.168.203.110@tcp LPNI seq info [58490:58490:8:4294967295]
00000100:00080000:0.0:1787763916.860662:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763916.860668:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:3.0:1787763916.860669:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763916.860672:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763916.860675:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763916.860679:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763916.860684:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21631
00000400:00000200:3.0:1787763916.860684:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000400:00000200:0.0:1787763916.860691:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:3.0:1787763916.860694:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787763916.860698:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763916.860701:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000010:3.0:1787763916.860701:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000100:00000040:0.0:1787763916.860706:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763916.860708:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1787763916.860712:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787763916.860714:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763916.860717:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:3.0:1787763916.860718:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000020:00000001:0.0:1787763916.860719:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763916.860722:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763916.860725:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:3.1:1787763916.860727:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:0.0:1787763916.860728:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763916.860733:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:3.1:1787763916.860734:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000020:00000001:0.0:1787763916.860736:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000100:3.0:1787763916.860741:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787763916.860743:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000010:3.0:1787763916.860745:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000100:00000001:0.0:1787763916.860747:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763916.860750:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:3.0:1787763916.860751:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1787763916.860752:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763916.860756:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:3.0:1787763916.860760:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:0.0:1787763916.860762:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763916.860766:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763916.860766:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763916.860771:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
02000000:00000001:0.0:1787763916.860772:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763916.860777:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763916.860778:0:257575:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:0.0:1787763916.860781:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763916.860785:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000200:3.0:1787763916.860786:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876ed00 x1874599832672640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763941 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1787763916.860788:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763916.860793:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763916.860797:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763916.860817:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000100:00000001:3.0:1787763916.860818:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000040:0.0:1787763916.860821:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000040:3.0:1787763916.860824:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832672640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763941 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763916.860825:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763916.860828:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763916.860832:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763916.860835:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1787763916.860838:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763916.860839:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763916.860843:0:257575:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000010:0.0:1787763916.860846:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:3.0:1787763916.860850:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763916.860851:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763916.860858:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00100000:0.0:1787763916.860861:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832672896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787763916.860863:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316305139
00000100:00000001:0.0:1787763916.860872:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000010:3.0:1787763916.860873:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d409c0.
00000100:00000001:0.0:1787763916.860874:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763916.860880:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832672896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.860885:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787763916.860886:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763916.860892:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763916.860894:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763916.860898:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787763916.860898:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316305139
00000100:00080000:0.0:1787763916.860906:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763916.860907:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000010:3.0:1787763916.860908:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40cc0.
00000100:00080000:0.0:1787763916.860909:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763916.860915:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787763916.860916:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00080000:0.0:1787763916.860918:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21631
00000100:00000001:3.0:1787763916.860925:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1787763916.860926:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763916.860932:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787763916.860933:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763916.860934:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763916.860940:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763916.860940:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:0.0:1787763916.860944:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763916.860946:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763916.860949:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763916.860951:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763916.860952:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000100:00100000:3.0:1787763916.860952:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763916/real 1787763916]  req@ffff9751f876ed00 x1874599832672640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787763916.860955:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763916.860957:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763916.860961:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763916.860964:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787763916.860970:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763916.860974:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763916.860978:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763916.860980:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000001:3.0:1787763916.860980:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763916.860983:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000020:00000040:0.0:1787763916.860984:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000400:00000010:3.0:1787763916.860987:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000100:00080000:0.0:1787763916.860989:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:3.0:1787763916.860992:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000100:00000001:0.0:1787763916.860993:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787763916.860997:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
02000000:00000001:0.0:1787763916.860998:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763916.861002:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763916.861003:0:257575:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:0.0:1787763916.861005:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763916.861008:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000200:3.0:1787763916.861010:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832672640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1787763916.861012:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763916.861017:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763916.861021:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787763916.861029:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832672640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:0.0:1787763916.861033:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000100:00000001:3.0:1787763916.861036:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000020:00000040:0.0:1787763916.861038:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:3.0:1787763916.861039:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787763916.861041:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1787763916.861043:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:0.0:1787763916.861044:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763916.861046:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763916.861047:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763916.861050:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763916.861054:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763916.861054:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832672640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1787763916.861060:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763916.861065:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763916.861066:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763916.861073:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832673024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:3.0:1787763916.861074:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832672640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763916.861095:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787763916.861097:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763916.861099:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763916.861102:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763916.861106:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:0.0:1787763916.861107:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832673024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861108:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763916.861114:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763916.861117:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763916.861121:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763916.861122:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000040:0.0:1787763916.861125:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21641)
00000100:00080000:3.0:1787763916.861128:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763916.861132:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763916.861155:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832672640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763916.861161:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832672640:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787763916.861165:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763916.861166:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763916.861168:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832672640/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861173:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763916.861175:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763916.861177:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610c00.
02000000:00000001:3.0:1787763916.861179:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763916.861180:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763916.861181:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787763916.861183:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763916.861185:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014400.
02000000:00000001:3.0:1787763916.861187:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763916.861188:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763916.861189:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:3.0:1787763916.861193:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763916.861194:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763916.861195:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763916.861200:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763916.861201:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763916.861203:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763916.861204:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763916.861205:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763916.861207:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f0c0 x1874599832672768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861216:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763916.861218:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763916.861219:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763916.861220:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763916.861222:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763916.861224:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832672768:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763916.861225:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763916.861227:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763916.861228:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763916.861229:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763916.861232:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610c00.
02000000:00000001:3.0:1787763916.861233:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763916.861236:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763916.861238:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763916.861240:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763916.861242:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832672768, portal 10
00000100:00000001:3.0:1787763916.861244:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763916.861245:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049530560 : -115096660021056 : ffff9751f876f0c0)
00000100:00000040:3.0:1787763916.861249:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f0c0 x1874599832672768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763941 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861254:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763916.861255:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763916.861257:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763916.861258:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832672768, offset 0
00000400:00000200:3.0:1787763916.861261:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763916.861264:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763916.861266:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763916.861268:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763916.861270:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763916.861272:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763916.861274:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763916.861276:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58491:58491:256:4294967295] 192.168.203.110@tcp LPNI seq info [58491:58491:8:4294967295]
00000400:00000200:3.0:1787763916.861279:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763916.861283:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763916.861287:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763916.861289:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763916.861292:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763916.861294:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787763916.861297:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763916.861299:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1787763916.861301:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763916.861303:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763916.861305:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763916.861307:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763916.861309:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763916.861310:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763916.861312:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763916.861314:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f0c0 x1874599832672768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763941 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861319:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763916.861321:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f0c0 x1874599832672768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763941 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861326:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763916.861327:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763916.861329:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763916.861332:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787763916.861334:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763916.861335:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316303904
00000100:00000001:3.0:1787763916.861337:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316305139
00000100:00000001:3.0:1787763916.861339:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316303904
00000100:00000001:3.0:1787763916.861340:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316305139
00000100:00000001:3.0:1787763916.861342:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763916.861344:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763916.861345:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763916.861347:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763916/real 1787763916]  req@ffff9751f876f0c0 x1874599832672768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763916.861352:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763916.861353:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763916.861354:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763916.861356:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763916.861357:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763916.861358:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763916.861360:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f0c0 x1874599832672768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763916.861365:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f0c0 x1874599832672768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861369:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763916.861370:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763916.861371:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763916.861372:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763916.861374:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f0c0 x1874599832672768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861378:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763916.861380:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f0c0 x1874599832672768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861384:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763916.861386:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763916.861387:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763916.861388:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763916.861389:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763916.861398:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763916.861399:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763916.861402:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763916.861403:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763916.861406:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f0c0 x1874599832672768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763916.861411:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832672768:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763916.861413:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763916.861415:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832672896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861419:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763916.861420:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763916.861421:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763916.861422:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763916.861423:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763916.861425:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832672896:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763916.861427:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763916.861428:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763916.861429:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763916.861430:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763916.861433:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787763916.861434:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763916.861436:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763916.861437:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763916.861439:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763916.861441:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832672896, portal 10
00000100:00000001:3.0:1787763916.861442:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763916.861443:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00000040:3.0:1787763916.861447:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832672896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763941 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861451:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763916.861453:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763916.861454:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763916.861455:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832672896, offset 0
00000400:00000200:3.0:1787763916.861458:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763916.861460:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763916.861462:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763916.861464:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763916.861465:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763916.861467:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763916.861468:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763916.861470:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58492:58492:256:4294967295] 192.168.203.110@tcp LPNI seq info [58492:58492:8:4294967295]
00000400:00000200:3.0:1787763916.861473:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763916.861477:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763916.861480:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763916.861481:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763916.861484:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763916.861486:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787763916.861488:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763916.861490:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1787763916.861492:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763916.861493:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763916.861495:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763916.861497:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763916.861499:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763916.861500:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763916.861501:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763916.861504:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e580 x1874599832672896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763941 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861509:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763916.861511:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832672896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763941 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861515:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763916.861516:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763916.861518:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763916.861519:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787763916.861521:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763916.861523:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316303892
00000100:00000001:3.0:1787763916.861524:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316305139
00000100:00000001:3.0:1787763916.861526:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316303892
00000100:00000001:3.0:1787763916.861527:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316305139
00000100:00000001:3.0:1787763916.861529:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763916.861530:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763916.861531:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763916.861533:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763916/real 1787763916]  req@ffff9751f876e580 x1874599832672896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763916.861537:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763916.861538:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763916.861539:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763916.861540:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763916.861541:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763916.861543:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763916.861545:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832672896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763916.861550:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832672896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861554:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763916.861555:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763916.861556:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763916.861556:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763916.861558:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832672896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861562:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763916.861565:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832672896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861569:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763916.861571:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763916.861572:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763916.861573:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763916.861574:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763916.861575:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763916.861576:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763916.861578:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763916.861579:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763916.861581:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832672896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763916.861586:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832672896:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763916.861588:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763916.861590:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832673024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861594:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763916.861595:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763916.861596:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763916.861597:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763916.861598:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763916.861600:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832673024:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763916.861601:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763916.861602:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763916.861603:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763916.861604:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763916.861606:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787763916.861608:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763916.861610:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763916.861611:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763916.861613:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763916.861615:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832673024, portal 4
00000100:00000001:3.0:1787763916.861616:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763916.861617:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000100:00000040:3.0:1787763916.861621:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832673024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763941 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861625:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763916.861626:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763916.861627:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763916.861629:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832673024, offset 0
00000400:00000200:3.0:1787763916.861631:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763916.861633:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763916.861635:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763916.861637:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763916.861638:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763916.861640:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763916.861642:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763916.861643:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58493:58493:256:4294967295] 192.168.203.110@tcp LPNI seq info [58493:58493:8:4294967295]
00000400:00000200:3.0:1787763916.861646:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763916.861650:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763916.861652:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763916.861654:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763916.861656:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763916.861657:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787763916.861660:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763916.861662:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1787763916.861664:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763916.861666:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763916.861667:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763916.861669:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763916.861671:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763916.861672:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763916.861674:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763916.861676:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e1c0 x1874599832673024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763941 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861681:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763916.861683:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832673024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763941 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861687:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763916.861688:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763916.861689:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763916.861691:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787763916.861693:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763916.861694:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316303909
00000100:00000001:3.0:1787763916.861696:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316305139
00000100:00000001:3.0:1787763916.861697:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316303909
00000100:00000001:3.0:1787763916.861699:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316305139
00000100:00000001:3.0:1787763916.861700:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763916.861701:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763916.861702:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763916.861704:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763916/real 1787763916]  req@ffff9751f876e1c0 x1874599832673024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763916.861708:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763916.861709:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763916.861710:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763916.861711:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763916.861712:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763916.861713:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763916.861715:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832673024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763916.861721:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832673024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861725:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763916.861726:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763916.861727:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763916.861728:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763916.861731:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832673024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861734:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763916.861737:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832673024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861741:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763916.861742:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763916.861743:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763916.861743:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763916.861745:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763916.861745:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763916.861747:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763916.861749:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763916.861750:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763916.861752:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832673024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763916.861757:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832673024:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763916.861759:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763916.861760:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763916.861762:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f0c0 x1874599832672768/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861766:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763916.861767:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763916.861768:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610c00.
02000000:00000001:3.0:1787763916.861770:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763916.861771:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763916.861772:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763916.861773:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763916.861774:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:3.0:1787763916.861776:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763916.861777:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763916.861777:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f0c0.
00000100:00000001:3.0:1787763916.861779:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763916.861780:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763916.861781:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763916.861783:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832672896/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861787:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763916.861788:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763916.861789:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787763916.861790:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763916.861791:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763916.861791:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763916.861792:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763916.861793:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
02000000:00000001:3.0:1787763916.861795:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763916.861795:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763916.861796:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:3.0:1787763916.861798:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763916.861798:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763916.861799:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763916.861818:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832673024/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763941 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763916.861823:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763916.861823:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763916.861824:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787763916.861826:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763916.861826:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763916.861827:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787763916.861829:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763916.861829:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017800.
02000000:00000001:3.0:1787763916.861832:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763916.861832:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763916.861833:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:3.0:1787763916.861835:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763916.861835:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763916.861836:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763916.861838:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763916.861839:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763916.861840:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763916.861842:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763917.500201:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763917.500202:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763917.500208:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763917.500209:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763917.500213:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763917.500215:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763917.500218:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763917.500218:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763917.500224:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763917.500224:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763917.500228:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763917.500228:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763917.500233:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763917.500235:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763917.500237:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763917.500286:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763917.500289:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763917.500292:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763917.884203:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763917.884209:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763917.884211:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763917.884216:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763917.884220:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763917.884222:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763917.884224:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763917.884226:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763917.884228:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763918.140217:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316305139, at 4316305152
00000100:00000010:3.0:1787763918.140231:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40cc0.
00000100:00000040:2.0:1787763918.140249:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316305139, at 4316305152
00000100:00000010:2.0:1787763918.140257:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d409c0.
00010000:00000001:3.0:1787763918.524217:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763918.524224:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763918.524229:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763918.524231:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763918.524234:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763918.524235:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763918.524235:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763918.524241:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763918.524243:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763918.524246:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763918.524249:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763918.524255:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763918.524258:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763918.524261:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763918.524264:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763918.524300:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763918.524304:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763918.524308:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763918.908226:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763918.908234:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763918.908237:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763918.908242:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763918.908246:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763918.908249:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763918.908251:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763918.908254:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763918.908257:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763919.548253:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763919.548264:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763919.548271:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763919.548273:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763919.548276:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763919.548278:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763919.548288:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763919.548294:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763919.548296:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763919.548300:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763919.548304:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763919.548307:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763919.548309:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763919.548312:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763919.548315:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763919.548355:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763919.548359:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763919.548362:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763919.932202:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763919.932208:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763919.932211:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763919.932217:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763919.932220:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763919.932225:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763919.932227:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763919.932229:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763919.932232:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763920.572228:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763920.572238:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763920.572248:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763920.572250:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763920.572255:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763920.572256:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763920.572263:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763920.572263:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763920.572267:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763920.572275:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763920.572279:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763920.572282:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763920.572284:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763920.572286:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763920.572289:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763920.572337:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763920.572340:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763920.572343:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763920.956249:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763920.956257:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763920.956261:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763920.956267:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763920.956271:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763920.956275:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763920.956277:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763920.956279:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763920.956281:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763921.596226:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763921.596234:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763921.596235:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763921.596243:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763921.596243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763921.596247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763921.596251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763921.596259:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763921.596260:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763921.596263:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763921.596270:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763921.596271:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763921.596282:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787763921.596283:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763921.596292:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763921.596294:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763921.596299:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763921.596301:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763921.596303:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763921.596305:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763921.596313:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763921.596403:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763921.596409:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763921.596413:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763921.980248:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763921.980257:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763921.980260:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763921.980266:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763921.980273:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763921.980278:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763921.980282:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763921.980284:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763921.980288:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763921.980298:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763921.980310:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787763921.980322:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763921.980327:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763921.980334:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763921.980344:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21636
00000400:00000200:1.0:1787763921.980354:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763921.980363:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763921.980368:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763921.980373:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763921.980376:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763921.980379:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763921.980382:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763921.980388:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763921.980390:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787763921.980393:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763921.980398:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787763921.980401:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787763921.980404:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787763921.980407:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763921.980409:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763921.980412:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763921.980414:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763921.980418:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787763921.980422:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763921.980424:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763921.980429:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763921.980434:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763921.980437:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763921.980440:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763921.980444:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763921.980456:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933840.
00000020:00000040:1.0:1787763921.980461:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787763921.980466:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763921.980468:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763921.980472:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763921.980475:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763921.980480:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763921.980492:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8800.
00000100:00000001:1.0:1787763921.980502:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763921.980514:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933840 x1874599832673152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763921.980527:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763921.980532:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763921.980538:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000083dd6d32] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933840 x1874599832673152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763921.980561:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763921.980565:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763921.980569:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787763921.980571:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763921.980573:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763921.980577:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763921.980581:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21636
00000100:00000001:3.0:1787763921.980584:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:1.0:1787763921.980586:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763921.980590:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000400:00000200:1.0:1787763921.980593:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787763921.980595:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00080000:1.0:1787763921.980600:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763921.980605:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933840 x1874599832673152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787763921.980607:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763921.980614:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787763921.980619:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:1.0:1787763921.980621:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763921.980622:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763921.980628:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:3.0:1787763921.980630:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763921.980633:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:1.0:1787763921.980634:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763921.980638:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
02000000:00000001:3.0:1787763921.980638:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763921.980643:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00100000:3.0:1787763921.980644:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832673152:192.168.203.110@tcp:250:kworker.0
00000020:00000040:1.0:1787763921.980649:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000001:3.0:1787763921.980649:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:1.0:1787763921.980655:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:3.0:1787763921.980655:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763921.980658:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763921.980661:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:3.0:1787763921.980662:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763921.980668:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
00000100:00000040:1.0:1787763921.980669:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:3.0:1787763921.980671:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763921.980676:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787763921.980680:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:1.0:1787763921.980682:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000010:3.0:1787763921.980685:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000100:00000001:1.0:1787763921.980686:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763921.980692:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000400:00000010:3.0:1787763921.980696:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00080000:1.0:1787763921.980697:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763921.980702:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763921.980705:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:3.0:1787763921.980707:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832673152, portal 25
00000100:00000001:3.0:1787763921.980710:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763921.980712:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497735744 : -115097211815872 : ffff9751d7933840)
02000000:00000001:1.0:1787763921.980713:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763921.980719:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763921.980719:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933840 x1874599832673152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763946 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763921.980725:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787763921.980730:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763921.980733:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000008:00100000:1.0:1787763921.980734:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000010:3.0:1787763921.980736:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763921.980740:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832673152, offset 0
00000008:00000001:1.0:1787763921.980742:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763921.980745:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763921.980748:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000010:1.0:1787763921.980751:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932580.
00000020:00000040:1.0:1787763921.980755:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763921.980758:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763921.980760:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763921.980762:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763921.980763:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763921.980781:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763921.980786:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca400.
00000400:00000200:3.0:1787763921.980787:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763921.980794:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763921.980798:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00100000:1.0:1787763921.980804:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932580 x1874599832673280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787763921.980807:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763921.980815:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763921.980822:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1787763921.980823:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1787763921.980826:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:1.0:1787763921.980828:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763921.980833:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58494:58494:256:4294967295] 192.168.203.110@tcp LPNI seq info [58494:58494:8:4294967295]
00000400:00000200:3.0:1787763921.980838:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:1.0:1787763921.980839:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d026924] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932580 x1874599832673280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787763921.980852:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:1.0:1787763921.980862:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787763921.980863:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00080000:1.0:1787763921.980870:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:3.0:1787763921.980871:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000100:00080000:1.0:1787763921.980878:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000800:00000200:3.0:1787763921.980882:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787763921.980883:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763921.980887:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:3.0:1787763921.980888:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000100:00000001:1.0:1787763921.980894:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:3.1:1787763921.980896:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00080000:1.0:1787763921.980901:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21636
00000800:00000200:3.1:1787763921.980902:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:3.0:1787763921.980910:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787763921.980912:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000010:3.0:1787763921.980913:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:1.0:1787763921.980919:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1787763921.980922:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:1.0:1787763921.980923:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763921.980931:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:3.0:1787763921.980931:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:1.0:1787763921.980938:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787763921.980939:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763921.980944:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000040:1.0:1787763921.980945:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763921.980950:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787763921.980950:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000020:00000001:1.0:1787763921.980956:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000200:3.0:1787763921.980956:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933840 x1874599832673152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763946 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787763921.980960:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787763921.980964:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763921.980969:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000100:00000001:3.0:1787763921.980971:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000001:1.0:1787763921.980973:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787763921.980978:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:3.0:1787763921.980978:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933840 x1874599832673152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763946 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787763921.980988:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763921.980990:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763921.980992:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787763921.980993:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:1.0:1787763921.980998:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763921.981001:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763921.981002:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:3.0:1787763921.981008:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:1.0:1787763921.981012:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00000001:3.0:1787763921.981012:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316310259
00000100:00080000:1.0:1787763921.981021:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000010:3.0:1787763921.981021:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40cc0.
00000100:00000001:1.0:1787763921.981027:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763921.981030:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787763921.981033:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
02000000:00000001:1.0:1787763921.981034:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763921.981039:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763921.981041:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316310259
00000100:00000001:1.0:1787763921.981042:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763921.981047:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000010:3.0:1787763921.981047:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40f60.
00000100:00000001:3.0:1787763921.981052:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000008:00000001:1.0:1787763921.981053:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763921.981055:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763921.981057:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763921.981059:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763921.981062:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000010:1.0:1787763921.981063:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930780.
00000100:00100000:3.0:1787763921.981067:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763921/real 1787763921]  req@ffff9751d7933840 x1874599832673152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787763921.981069:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763921.981077:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763921.981080:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1787763921.981080:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763921.981083:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:1.0:1787763921.981086:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:3.0:1787763921.981087:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
02000000:00000001:1.0:1787763921.981091:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1787763921.981092:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763921.981095:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
02000000:00000001:1.0:1787763921.981097:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763921.981098:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763921.981103:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933840 x1874599832673152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1787763921.981105:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8c00.
00000100:00000200:3.0:1787763921.981114:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933840 x1874599832673152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763921.981115:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763921.981124:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763921.981126:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787763921.981127:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930780 x1874599832673408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787763921.981129:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763921.981131:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763921.981166:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00100000:3.0:1787763921.981171:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933840 x1874599832673152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763921.981172:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763921.981183:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763921.981184:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3977950] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930780 x1874599832673408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787763921.981190:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933840 x1874599832673152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763921.981208:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763921.981208:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763921.981214:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763921.981215:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787763921.981216:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763921.981218:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763921.981223:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000040:1.0:1787763921.981225:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1787763921.981226:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763921.981231:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787763921.981233:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787763921.981236:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763921.981242:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00080000:1.0:1787763921.981243:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787763921.981247:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:3.0:1787763921.981249:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933840 x1874599832673152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787763921.981251:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763921.981259:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00100000:3.0:1787763921.981260:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832673152:192.168.203.110@tcp:250:kworker.0
00000100:00080000:1.0:1787763921.981264:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21636
00000400:00000200:1.0:1787763921.981270:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763921.981274:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763921.981276:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763921.981281:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763921.981283:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763921.981286:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787763921.981286:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763921.981293:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933840 x1874599832673152/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787763921.981295:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763921.981304:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787763921.981304:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763921.981307:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000020:00000001:1.0:1787763921.981311:0:304425:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000010:3.0:1787763921.981312:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
00000020:00000040:1.0:1787763921.981316:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
02000000:00000001:3.0:1787763921.981317:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763921.981320:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000001:1.0:1787763921.981321:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763921.981323:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000040:1.0:1787763921.981326:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:3.0:1787763921.981326:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763921.981330:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8800.
00000020:00000001:1.0:1787763921.981332:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
02000000:00000001:3.0:1787763921.981336:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000020:00000001:1.0:1787763921.981337:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
02000000:00000001:3.0:1787763921.981339:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763921.981343:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933840.
00000100:00000040:1.0:1787763921.981346:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763921.981349:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787763921.981350:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787763921.981351:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763921.981355:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787763921.981356:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763921.981359:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763921.981362:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:3.0:1787763921.981363:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:1.0:1787763921.981367:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763921.981368:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763921.981372:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787763921.981375:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763921.981375:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763921.981379:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
02000000:00000001:1.0:1787763921.981380:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:3.0:1787763921.981386:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932580 x1874599832673280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787763921.981387:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763921.981393:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763921.981399:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763921.981404:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1787763921.981404:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763921.981407:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:1.0:1787763921.981409:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763921.981412:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:1.0:1787763921.981414:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763921.981416:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000010:1.0:1787763921.981420:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932940.
02000000:00000001:3.0:1787763921.981421:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1787763921.981427:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00100000:3.0:1787763921.981429:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832673280:192.168.203.110@tcp:38:kworker.0
02000000:00000001:1.0:1787763921.981435:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1787763921.981435:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763921.981438:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787763921.981439:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763921.981440:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763921.981445:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1787763921.981447:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000010:3.0:1787763921.981449:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:1.0:1787763921.981450:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763921.981452:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763921.981455:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763921.981459:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763921.981463:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
02000000:00000010:1.0:1787763921.981464:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8800.
00000400:00000010:3.0:1787763921.981467:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000001:1.0:1787763921.981470:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787763921.981471:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832673280, portal 10
00000100:00000001:3.0:1787763921.981476:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763921.981478:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497730944 : -115097211820672 : ffff9751d7932580)
00000100:00100000:1.0:1787763921.981483:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932940 x1874599832673536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:3.0:1787763921.981488:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932580 x1874599832673280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763946 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.981505:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:1.0:1787763921.981507:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:3.0:1787763921.981508:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763921.981512:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000001:1.0:1787763921.981514:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787763921.981515:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832673280, offset 0
00000400:00000200:3.0:1787763921.981522:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000040:1.0:1787763921.981527:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000af90eaa] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932940 x1874599832673536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787763921.981527:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763921.981533:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763921.981538:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763921.981543:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763921.981547:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1787763921.981551:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763921.981551:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763921.981554:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58495:58495:256:4294967295] 192.168.203.110@tcp LPNI seq info [58495:58495:8:4294967295]
00000400:00000200:3.0:1787763921.981559:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:1.0:1787763921.981560:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21646)
00000400:00000200:3.0:1787763921.981568:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763921.981571:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763921.981574:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763921.981577:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763921.981579:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787763921.981584:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763921.981585:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:3.0:1787763921.981589:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763921.981590:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763921.981592:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763921.981595:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763921.981597:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763921.981598:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763921.981600:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763921.981603:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932580 x1874599832673280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763946 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.981609:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763921.981611:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932580 x1874599832673280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763946 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.981615:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763921.981616:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763921.981619:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763921.981623:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787763921.981625:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763921.981627:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316309024
00000100:00000001:3.0:1787763921.981629:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316310259
00000100:00000001:3.0:1787763921.981631:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316309024
00000100:00000001:3.0:1787763921.981633:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316310259
00000100:00000001:3.0:1787763921.981634:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763921.981636:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763921.981637:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763921.981639:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763921/real 1787763921]  req@ffff9751d7932580 x1874599832673280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763921.981644:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763921.981645:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763921.981646:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763921.981647:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763921.981649:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763921.981650:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763921.981652:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932580 x1874599832673280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763921.981658:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932580 x1874599832673280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.981661:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763921.981662:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763921.981663:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763921.981664:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763921.981666:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932580 x1874599832673280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.981670:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763921.981673:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932580 x1874599832673280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.981679:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763921.981681:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763921.981682:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763921.981683:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763921.981684:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763921.981694:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763921.981695:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763921.981697:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763921.981699:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763921.981701:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932580 x1874599832673280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763921.981707:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832673280:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763921.981710:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763921.981712:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930780 x1874599832673408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.981716:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763921.981717:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763921.981718:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763921.981719:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763921.981720:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763921.981722:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832673408:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763921.981724:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763921.981725:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763921.981726:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763921.981727:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763921.981729:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787763921.981731:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763921.981733:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763921.981735:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763921.981737:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763921.981739:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832673408, portal 10
00000100:00000001:3.0:1787763921.981740:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763921.981741:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497723264 : -115097211828352 : ffff9751d7930780)
00000100:00000040:3.0:1787763921.981746:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930780 x1874599832673408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763946 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.981750:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763921.981752:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763921.981753:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763921.981755:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832673408, offset 0
00000400:00000200:3.0:1787763921.981757:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763921.981759:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763921.981761:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763921.981763:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763921.981765:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763921.981780:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763921.981782:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763921.981786:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58496:58496:256:4294967295] 192.168.203.110@tcp LPNI seq info [58496:58496:8:4294967295]
00000400:00000200:3.0:1787763921.981790:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763921.981796:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763921.981801:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763921.981804:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763921.981808:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763921.981811:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787763921.981816:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763921.981818:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:3.0:1787763921.981822:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763921.981823:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763921.981825:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763921.981828:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763921.981830:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763921.981832:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763921.981834:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763921.981837:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930780 x1874599832673408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763946 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.981843:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763921.981845:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930780 x1874599832673408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763946 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.981849:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763921.981850:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763921.981852:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763921.981854:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787763921.981856:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763921.981857:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316309012
00000100:00000001:3.0:1787763921.981859:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316310259
00000100:00000001:3.0:1787763921.981861:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316309012
00000100:00000001:3.0:1787763921.981862:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316310259
00000100:00000001:3.0:1787763921.981864:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763921.981865:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763921.981866:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763921.981869:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763921/real 1787763921]  req@ffff9751d7930780 x1874599832673408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763921.981873:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763921.981874:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763921.981875:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763921.981877:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763921.981878:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763921.981879:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763921.981881:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930780 x1874599832673408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763921.981886:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930780 x1874599832673408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.981891:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763921.981892:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763921.981894:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763921.981895:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763921.981897:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930780 x1874599832673408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.981901:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763921.981903:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930780 x1874599832673408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.981907:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763921.981909:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763921.981910:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763921.981910:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763921.981911:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763921.981912:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763921.981914:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763921.981916:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763921.981917:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763921.981919:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930780 x1874599832673408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763921.981924:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832673408:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763921.981926:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763921.981927:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763921.981930:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932580 x1874599832673280/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.981934:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763921.981936:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763921.981937:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787763921.981939:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763921.981940:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763921.981941:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763921.981943:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763921.981944:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca400.
02000000:00000001:3.0:1787763921.981946:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763921.981947:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763921.981948:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932580.
00000100:00000001:3.0:1787763921.981950:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763921.981951:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763921.981952:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763921.981954:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930780 x1874599832673408/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.981958:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763921.981959:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763921.981960:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787763921.981962:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763921.981962:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763921.981963:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763921.981964:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763921.981965:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8c00.
02000000:00000001:3.0:1787763921.981966:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763921.981967:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763921.981968:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930780.
00000100:00000001:3.0:1787763921.981970:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763921.981971:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763921.981972:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763921.981975:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763921.981975:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763921.981977:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763921.981978:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763921.981979:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763921.981981:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932940 x1874599832673536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.981986:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763921.981987:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763921.981989:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763921.981990:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763921.981992:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763921.981994:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832673536:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763921.981995:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763921.981997:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763921.981998:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763921.981999:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763921.982001:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787763921.982002:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763921.982004:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763921.982006:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763921.982009:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763921.982011:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832673536, portal 4
00000100:00000001:3.0:1787763921.982012:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763921.982013:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497731904 : -115097211819712 : ffff9751d7932940)
00000100:00000040:3.0:1787763921.982017:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932940 x1874599832673536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763946 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.982021:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763921.982023:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763921.982024:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763921.982026:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832673536, offset 0
00000400:00000200:3.0:1787763921.982028:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763921.982030:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763921.982032:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763921.982034:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763921.982036:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763921.982038:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763921.982040:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763921.982042:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58497:58497:256:4294967295] 192.168.203.110@tcp LPNI seq info [58497:58497:8:4294967295]
00000400:00000200:3.0:1787763921.982045:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763921.982049:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763921.982052:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763921.982054:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763921.982057:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763921.982059:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:3.1:1787763921.982062:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763921.982064:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:3.0:1787763921.982066:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763921.982068:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763921.982070:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763921.982072:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763921.982074:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763921.982075:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763921.982077:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763921.982079:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932940 x1874599832673536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763946 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.982084:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763921.982086:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932940 x1874599832673536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763946 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.982090:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763921.982091:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763921.982093:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763921.982095:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787763921.982096:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763921.982097:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316309029
00000100:00000001:3.0:1787763921.982099:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316310259
00000100:00000001:3.0:1787763921.982101:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316309029
00000100:00000001:3.0:1787763921.982102:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316310259
00000100:00000001:3.0:1787763921.982104:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763921.982105:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763921.982106:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763921.982108:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763921/real 1787763921]  req@ffff9751d7932940 x1874599832673536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763921.982114:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763921.982115:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763921.982116:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763921.982118:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763921.982119:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763921.982120:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763921.982121:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932940 x1874599832673536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763921.982126:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932940 x1874599832673536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.982130:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763921.982131:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763921.982132:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763921.982148:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763921.982151:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932940 x1874599832673536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.982156:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763921.982158:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932940 x1874599832673536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.982163:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763921.982165:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763921.982166:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763921.982166:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763921.982167:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763921.982168:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763921.982170:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763921.982172:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763921.982173:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763921.982176:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932940 x1874599832673536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763921.982182:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832673536:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763921.982184:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763921.982185:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763921.982187:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932940 x1874599832673536/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763946 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763921.982191:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763921.982192:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763921.982193:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787763921.982194:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763921.982195:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763921.982196:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787763921.982197:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763921.982198:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8800.
02000000:00000001:3.0:1787763921.982199:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763921.982200:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763921.982201:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932940.
00000100:00000001:3.0:1787763921.982203:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763921.982203:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763921.982205:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763921.982207:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763921.982208:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763921.982209:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763921.982210:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763922.620250:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763922.620256:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763922.620259:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763922.620265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763922.620273:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763922.620274:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763922.620280:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763922.620282:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763922.620288:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763922.620290:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763922.620293:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763922.620294:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763922.620300:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763922.620304:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763922.620307:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763922.620386:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763922.620390:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763922.620394:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763923.004241:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763923.004250:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763923.004253:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763923.004259:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763923.004263:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763923.004268:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763923.004271:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763923.004274:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763923.004278:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763923.260272:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316310259, at 4316310272
00000100:00000010:3.0:1787763923.260293:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40f60.
00000100:00000040:2.0:1787763923.260336:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316310259, at 4316310272
00000100:00000010:2.0:1787763923.260356:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40cc0.
00010000:00000001:3.0:1787763923.644212:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763923.644221:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763923.644224:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763923.644227:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763923.644233:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763923.644236:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763923.644238:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763923.644239:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763923.644244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763923.644246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763923.644250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763923.644253:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763923.644253:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763923.644261:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763923.644263:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763923.644270:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763923.644274:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763923.644279:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763923.644282:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763923.644285:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763923.644290:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763923.644314:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763923.644317:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763923.644321:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763924.028209:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763924.028216:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763924.028218:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763924.028222:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763924.028225:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763924.028228:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763924.028230:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763924.028232:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763924.028234:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763924.668251:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763924.668258:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763924.668259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763924.668260:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763924.668264:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763924.668266:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763924.668269:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763924.668271:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763924.668272:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763924.668275:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763924.668277:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763924.668280:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763924.668283:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763924.668286:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763924.668288:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763924.668342:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763924.668345:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763924.668349:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763925.052204:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763925.052212:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763925.052213:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763925.052217:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763925.052221:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763925.052224:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763925.052226:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763925.052228:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763925.052231:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763925.692226:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763925.692227:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763925.692232:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763925.692233:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763925.692236:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763925.692238:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763925.692241:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763925.692241:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763925.692248:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763925.692249:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763925.692253:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763925.692253:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763925.692259:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763925.692261:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763925.692263:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763925.692305:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763925.692310:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763925.692313:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763926.076250:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763926.076257:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763926.076259:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763926.076264:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763926.076268:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763926.076272:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763926.076276:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763926.076280:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763926.076283:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763926.716232:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763926.716237:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763926.716238:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763926.716242:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763926.716243:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763926.716248:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763926.716252:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763926.716253:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763926.716258:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763926.716258:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763926.716263:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787763926.716264:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763926.716268:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763926.716269:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763926.716275:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787763926.716276:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763926.716288:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763926.716291:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763926.716295:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763926.716299:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763926.716300:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763926.716357:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763926.716362:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763926.716365:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763927.100239:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763927.100253:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763927.100256:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763927.100263:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763927.100267:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:1.0:1787763927.100268:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763927.100270:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763927.100275:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:1.0:1787763927.100278:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787763927.100278:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763927.100283:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787763927.100284:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763927.100288:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763927.100294:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763927.100302:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21641
00000400:00000200:1.0:1787763927.100312:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763927.100320:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763927.100324:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763927.100328:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763927.100333:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763927.100337:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763927.100340:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763927.100346:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763927.100349:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787763927.100352:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763927.100357:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787763927.100360:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787763927.100362:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787763927.100365:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763927.100368:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763927.100371:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763927.100373:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763927.100376:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787763927.100379:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763927.100382:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763927.100385:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763927.100388:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763927.100392:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763927.100394:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763927.100397:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763927.100406:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933480.
00000020:00000040:1.0:1787763927.100410:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787763927.100414:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763927.100417:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763927.100419:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763927.100422:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763927.100426:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763927.100434:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb800.
00000100:00000001:1.0:1787763927.100440:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763927.100449:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933480 x1874599832673664/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763927.100460:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763927.100464:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763927.100470:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3c2c33c] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933480 x1874599832673664/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763927.100492:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763927.100495:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763927.100499:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787763927.100500:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763927.100502:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763927.100505:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763927.100508:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21641
00000400:00000200:1.0:1787763927.100513:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763927.100518:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763927.100521:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787763927.100523:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:1.0:1787763927.100525:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763927.100529:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763927.100530:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763927.100536:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763927.100536:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1787763927.100541:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763927.100545:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763927.100547:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00000040:3.0:1787763927.100548:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933480 x1874599832673664/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1787763927.100549:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763927.100553:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787763927.100558:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787763927.100563:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787763927.100570:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763927.100571:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763927.100575:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787763927.100576:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763927.100580:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763927.100584:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:3.0:1787763927.100584:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763927.100589:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:1.0:1787763927.100590:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:3.0:1787763927.100597:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763927.100598:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763927.100604:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763927.100607:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832673664:192.168.203.110@tcp:250:kworker.0
02000000:00000001:1.0:1787763927.100608:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787763927.100614:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787763927.100616:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763927.100620:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763927.100620:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1787763927.100625:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1787763927.100625:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:1.0:1787763927.100632:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1787763927.100632:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:1.0:1787763927.100638:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763927.100641:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763927.100641:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787763927.100648:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763927.100650:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930000.
00000020:00000040:1.0:1787763927.100655:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000200:3.0:1787763927.100659:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:1.0:1787763927.100661:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763927.100664:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763927.100667:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:3.0:1787763927.100668:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
02000000:00000001:1.0:1787763927.100669:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763927.100673:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763927.100680:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca800.
00000400:00000010:3.0:1787763927.100680:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000001:1.0:1787763927.100688:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787763927.100691:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832673664, portal 25
00000100:00100000:1.0:1787763927.100696:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930000 x1874599832673792/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787763927.100697:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763927.100701:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497734784 : -115097211816832 : ffff9751d7933480)
00000100:00000001:1.0:1787763927.100713:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:3.0:1787763927.100714:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933480 x1874599832673664/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763952 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763927.100717:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763927.100728:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6d90274] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930000 x1874599832673792/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.100744:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763927.100753:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763927.100760:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000001:1.0:1787763927.100763:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787763927.100765:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832673664, offset 0
00000100:00080000:1.0:1787763927.100768:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763927.100774:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787763927.100778:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1787763927.100779:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00080000:1.0:1787763927.100781:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763927.100786:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763927.100791:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21641
00000400:00000200:3.0:1787763927.100792:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763927.100800:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763927.100805:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787763927.100809:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763927.100814:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1787763927.100814:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:1.0:1787763927.100820:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:3.0:1787763927.100825:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:1.0:1787763927.100826:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763927.100833:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:3.0:1787763927.100834:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1787763927.100839:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:3.0:1787763927.100842:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000020:00000001:1.0:1787763927.100845:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763927.100849:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000400:00000200:3.0:1787763927.100851:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58498:58498:256:4294967295] 192.168.203.110@tcp LPNI seq info [58498:58498:8:4294967295]
00000020:00000001:1.0:1787763927.100852:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763927.100857:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787763927.100863:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:3.0:1787763927.100863:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:1.0:1787763927.100868:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787763927.100875:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763927.100880:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787763927.100885:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:1.0:1787763927.100886:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763927.100893:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763927.100897:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000800:00000200:3.0:1787763927.100899:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00080000:1.0:1787763927.100903:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763927.100908:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1787763927.100909:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
02000000:00000001:1.0:1787763927.100911:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763927.100916:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:3.0:1787763927.100922:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:1.0:1787763927.100923:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763927.100927:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:3.0:1787763927.100931:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000008:00100000:1.0:1787763927.100932:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763927.100939:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763927.100942:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763927.100946:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000010:1.0:1787763927.100950:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930b40.
00000800:00000200:3.1:1787763927.100952:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000020:00000040:1.0:1787763927.100955:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763927.100961:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000100:3.0:1787763927.100962:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
02000000:00000001:1.0:1787763927.100963:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1787763927.100967:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000100:00000001:1.0:1787763927.100968:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763927.100972:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763927.100976:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763927.100977:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000010:1.0:1787763927.100981:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8000.
00000400:00000200:3.0:1787763927.100986:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:1.0:1787763927.100989:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763927.100995:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000100:00100000:1.0:1787763927.100997:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930b40 x1874599832673920/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1787763927.101000:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763927.101007:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:1.0:1787763927.101011:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763927.101016:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787763927.101017:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933480 x1874599832673664/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763952 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787763927.101024:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022d7f91a] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930b40 x1874599832673920/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.101040:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787763927.101042:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763927.101046:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1787763927.101050:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933480 x1874599832673664/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763952 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787763927.101052:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787763927.101057:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763927.101065:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787763927.101067:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763927.101071:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763927.101072:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763927.101076:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787763927.101077:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:1.0:1787763927.101082:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21641
00000100:00000001:3.0:1787763927.101087:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787763927.101088:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763927.101096:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1787763927.101098:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:1.0:1787763927.101100:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787763927.101105:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316315379
00000100:00000040:1.0:1787763927.101106:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763927.101111:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763927.101114:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000010:3.0:1787763927.101114:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40f60.
00000100:00000001:1.0:1787763927.101117:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763927.101121:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763927.101122:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787763927.101124:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787763927.101126:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000020:00000040:1.0:1787763927.101127:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787763927.101132:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787763927.101158:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:1.0:1787763927.101162:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763927.101165:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316315380
00000100:00000001:1.0:1787763927.101166:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763927.101171:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000010:3.0:1787763927.101173:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40660.
00000100:00000001:1.0:1787763927.101175:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763927.101181:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:3.0:1787763927.101181:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00080000:1.0:1787763927.101187:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763927.101189:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787763927.101191:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763927.101196:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787763927.101197:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763927.101200:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763927.101202:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000001:1.0:1787763927.101205:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763927.101209:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763927.101213:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00100000:3.0:1787763927.101213:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763927/real 1787763927]  req@ffff9751d7933480 x1874599832673664/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:1.0:1787763927.101219:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763927.101223:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763927.101229:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7931680.
00000020:00000040:1.0:1787763927.101234:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000400:00000001:3.0:1787763927.101236:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:1.0:1787763927.101238:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000001:3.0:1787763927.101240:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:1.0:1787763927.101242:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787763927.101245:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000100:00000001:1.0:1787763927.101246:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763927.101249:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763927.101252:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763927.101252:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763927.101258:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
02000000:00000010:1.0:1787763927.101261:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9000.
00000100:00000001:3.0:1787763927.101264:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:1.0:1787763927.101266:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787763927.101272:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933480 x1874599832673664/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787763927.101276:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7931680 x1874599832674048/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763927.101300:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:3.0:1787763927.101301:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933480 x1874599832673664/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763927.101305:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763927.101316:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d4255d6] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7931680 x1874599832674048/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.101318:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763927.101322:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763927.101329:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763927.101333:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763927.101334:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763927.101342:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21651)
00000100:00100000:3.0:1787763927.101343:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933480 x1874599832673664/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.101354:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763927.101361:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933480 x1874599832673664/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.101370:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763927.101373:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763927.101375:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763927.101377:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763927.101379:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763927.101381:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763927.101384:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763927.101390:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763927.101392:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763927.101397:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933480 x1874599832673664/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763927.101407:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832673664:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787763927.101412:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763927.101414:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763927.101417:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933480 x1874599832673664/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.101425:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763927.101427:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763927.101430:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787763927.101433:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763927.101435:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763927.101437:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787763927.101439:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763927.101442:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb800.
02000000:00000001:3.0:1787763927.101446:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763927.101448:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763927.101450:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933480.
00000100:00000001:3.0:1787763927.101454:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763927.101455:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763927.101458:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763927.101465:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763927.101466:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763927.101468:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763927.101470:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763927.101472:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763927.101476:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930000 x1874599832673792/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.101484:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763927.101485:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763927.101488:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763927.101490:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763927.101493:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763927.101496:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832673792:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763927.101499:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763927.101502:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763927.101503:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763927.101505:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763927.101509:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787763927.101511:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763927.101515:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763927.101518:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763927.101522:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763927.101525:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832673792, portal 10
00000100:00000001:3.0:1787763927.101527:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763927.101529:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497721344 : -115097211830272 : ffff9751d7930000)
00000100:00000040:3.0:1787763927.101536:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930000 x1874599832673792/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763952 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.101543:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763927.101545:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763927.101548:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763927.101551:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832673792, offset 0
00000400:00000200:3.0:1787763927.101555:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763927.101560:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763927.101564:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763927.101567:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763927.101570:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763927.101573:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763927.101576:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763927.101579:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58499:58499:256:4294967295] 192.168.203.110@tcp LPNI seq info [58499:58499:8:4294967295]
00000400:00000200:3.0:1787763927.101583:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763927.101590:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763927.101596:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763927.101600:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763927.101604:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763927.101607:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1787763927.101613:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763927.101616:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:3.0:1787763927.101620:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763927.101622:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763927.101625:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763927.101629:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763927.101632:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763927.101634:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763927.101637:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763927.101641:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930000 x1874599832673792/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763952 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.101650:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763927.101653:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930000 x1874599832673792/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763952 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.101660:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763927.101662:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763927.101665:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763927.101669:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787763927.101673:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763927.101675:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316314144
00000100:00000001:3.0:1787763927.101678:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316315379
00000100:00000001:3.0:1787763927.101680:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316314144
00000100:00000001:3.0:1787763927.101682:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316315380
00000100:00000001:3.0:1787763927.101685:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763927.101687:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763927.101689:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763927.101694:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763927/real 1787763927]  req@ffff9751d7930000 x1874599832673792/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763927.101702:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763927.101704:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763927.101706:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763927.101708:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763927.101711:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763927.101713:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763927.101716:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930000 x1874599832673792/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763927.101725:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930000 x1874599832673792/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.101746:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763927.101747:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763927.101750:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763927.101751:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763927.101756:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930000 x1874599832673792/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.101764:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763927.101768:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930000 x1874599832673792/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.101776:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763927.101779:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763927.101781:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763927.101782:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763927.101784:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763927.101797:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763927.101801:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763927.101805:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763927.101807:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763927.101812:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930000 x1874599832673792/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763927.101821:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832673792:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763927.101825:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763927.101829:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930b40 x1874599832673920/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.101836:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763927.101838:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763927.101840:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763927.101842:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763927.101844:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763927.101848:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832673920:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763927.101851:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763927.101853:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763927.101854:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763927.101856:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763927.101861:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787763927.101863:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763927.101866:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763927.101869:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763927.101873:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763927.101876:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832673920, portal 10
00000100:00000001:3.0:1787763927.101878:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763927.101879:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497724224 : -115097211827392 : ffff9751d7930b40)
00000100:00000040:3.0:1787763927.101885:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930b40 x1874599832673920/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763952 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.101892:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763927.101895:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763927.101898:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763927.101900:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832673920, offset 0
00000400:00000200:3.0:1787763927.101905:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763927.101909:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763927.101913:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763927.101916:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763927.101919:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763927.101921:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763927.101924:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763927.101927:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58500:58500:256:4294967295] 192.168.203.110@tcp LPNI seq info [58500:58500:8:4294967295]
00000400:00000200:3.0:1787763927.101932:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763927.101939:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763927.101944:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763927.101948:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763927.101952:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763927.101955:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1787763927.101959:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763927.101962:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:3.0:1787763927.101965:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763927.101967:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763927.101970:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763927.101973:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763927.101976:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763927.101978:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763927.101981:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763927.101985:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930b40 x1874599832673920/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763952 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.101993:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763927.101996:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930b40 x1874599832673920/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763952 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.102003:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763927.102005:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763927.102008:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763927.102011:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787763927.102014:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763927.102017:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316314132
00000100:00000001:3.0:1787763927.102020:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316315379
00000100:00000001:3.0:1787763927.102022:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316314132
00000100:00000001:3.0:1787763927.102024:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316315380
00000100:00000001:3.0:1787763927.102027:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763927.102029:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763927.102031:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763927.102035:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763927/real 1787763927]  req@ffff9751d7930b40 x1874599832673920/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763927.102044:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763927.102046:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763927.102047:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763927.102050:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763927.102052:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763927.102054:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763927.102057:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930b40 x1874599832673920/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763927.102066:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930b40 x1874599832673920/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.102074:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763927.102076:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763927.102077:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763927.102079:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763927.102083:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930b40 x1874599832673920/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.102091:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763927.102095:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930b40 x1874599832673920/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.102103:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763927.102106:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763927.102108:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763927.102109:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763927.102111:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763927.102113:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763927.102116:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763927.102119:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763927.102122:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763927.102126:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930b40 x1874599832673920/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763927.102155:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832673920:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763927.102160:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763927.102164:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7931680 x1874599832674048/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.102171:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763927.102173:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763927.102175:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763927.102178:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763927.102180:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763927.102183:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832674048:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763927.102186:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763927.102188:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763927.102190:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763927.102191:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763927.102195:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610c00.
02000000:00000001:3.0:1787763927.102197:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763927.102200:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763927.102203:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763927.102207:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763927.102210:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832674048, portal 4
00000100:00000001:3.0:1787763927.102212:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763927.102214:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497727104 : -115097211824512 : ffff9751d7931680)
00000100:00000040:3.0:1787763927.102220:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7931680 x1874599832674048/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763952 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.102228:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763927.102230:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763927.102233:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763927.102235:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832674048, offset 0
00000400:00000200:3.0:1787763927.102239:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763927.102243:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763927.102247:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763927.102250:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763927.102253:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763927.102256:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763927.102258:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763927.102261:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58501:58501:256:4294967295] 192.168.203.110@tcp LPNI seq info [58501:58501:8:4294967295]
00000400:00000200:3.0:1787763927.102266:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763927.102272:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763927.102277:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763927.102280:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763927.102285:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763927.102288:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:3.1:1787763927.102292:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763927.102295:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:3.0:1787763927.102300:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763927.102302:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763927.102305:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763927.102309:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763927.102311:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763927.102313:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763927.102316:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763927.102320:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7931680 x1874599832674048/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763952 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.102335:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763927.102339:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7931680 x1874599832674048/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763952 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.102346:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763927.102348:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763927.102351:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763927.102354:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787763927.102357:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763927.102360:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316314150
00000100:00000001:3.0:1787763927.102362:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316315379
00000100:00000001:3.0:1787763927.102364:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316314150
00000100:00000001:3.0:1787763927.102366:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316315380
00000100:00000001:3.0:1787763927.102369:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763927.102371:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763927.102373:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763927.102377:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763927/real 1787763927]  req@ffff9751d7931680 x1874599832674048/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763927.102386:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763927.102387:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763927.102389:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763927.102391:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763927.102393:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763927.102396:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763927.102399:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7931680 x1874599832674048/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763927.102408:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7931680 x1874599832674048/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.102416:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763927.102418:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763927.102420:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763927.102421:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763927.102425:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7931680 x1874599832674048/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.102432:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763927.102436:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7931680 x1874599832674048/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.102444:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763927.102447:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763927.102448:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763927.102450:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763927.102452:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763927.102454:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763927.102457:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763927.102460:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763927.102463:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763927.102468:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7931680 x1874599832674048/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763927.102477:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832674048:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763927.102481:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763927.102483:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763927.102487:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930000 x1874599832673792/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.102494:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763927.102495:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763927.102498:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787763927.102500:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763927.102502:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763927.102504:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763927.102507:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763927.102508:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca800.
02000000:00000001:3.0:1787763927.102511:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763927.102512:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763927.102514:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930000.
00000100:00000001:3.0:1787763927.102517:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763927.102518:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763927.102521:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763927.102524:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930b40 x1874599832673920/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.102531:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763927.102533:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763927.102534:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787763927.102537:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763927.102538:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763927.102539:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763927.102541:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763927.102543:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8000.
02000000:00000001:3.0:1787763927.102545:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763927.102547:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763927.102548:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930b40.
00000100:00000001:3.0:1787763927.102550:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763927.102552:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763927.102554:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763927.102557:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7931680 x1874599832674048/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763952 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763927.102564:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763927.102566:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763927.102567:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610c00.
02000000:00000001:3.0:1787763927.102569:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763927.102571:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763927.102572:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787763927.102574:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763927.102576:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9000.
02000000:00000001:3.0:1787763927.102578:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763927.102580:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763927.102581:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7931680.
00000100:00000001:3.0:1787763927.102584:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763927.102585:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763927.102587:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763927.102591:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763927.102592:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763927.102595:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763927.102597:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763927.740220:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763927.740229:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763927.740231:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763927.740232:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763927.740237:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763927.740240:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763927.740244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763927.740247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763927.740251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763927.740252:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763927.740256:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763927.740262:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763927.740265:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763927.740268:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763927.740272:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763927.740320:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763927.740325:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763927.740329:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763928.124322:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763928.124331:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763928.124333:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763928.124338:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763928.124342:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763928.124345:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763928.124347:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763928.124349:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763928.124351:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763928.380239:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316315380, at 4316315392
00000100:00000010:3.0:1787763928.380254:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40660.
00000100:00000040:2.0:1787763928.380280:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316315379, at 4316315392
00000100:00000010:2.0:1787763928.380291:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40f60.
00000100:00000001:2.0:1787763928.764242:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763928.764247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763928.764250:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763928.764252:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763928.764255:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763928.764260:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763928.764261:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763928.764263:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763928.764267:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763928.764270:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763928.764273:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763928.764274:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763928.764281:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763928.764285:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763928.764289:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763928.764336:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763928.764347:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763928.764351:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763929.148240:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763929.148248:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763929.148251:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763929.148258:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763929.148263:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763929.148266:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763929.148270:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763929.148272:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763929.148275:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763929.788208:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763929.788214:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763929.788215:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763929.788217:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763929.788221:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763929.788222:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763929.788228:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763929.788231:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763929.788231:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763929.788237:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763929.788237:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763929.788240:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763929.788243:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763929.788244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763929.788248:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763929.788311:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763929.788315:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763929.788319:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763930.172269:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763930.172277:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763930.172280:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763930.172286:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763930.172291:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763930.172295:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763930.172299:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763930.172302:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763930.172305:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763930.812257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763930.812264:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763930.812267:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763930.812272:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763930.812276:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763930.812280:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763930.812281:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763930.812287:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763930.812288:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763930.812293:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763930.812296:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763930.812299:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763930.812299:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763930.812305:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763930.812305:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763930.812385:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763930.812390:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763930.812395:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763931.196223:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763931.196231:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763931.196234:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763931.196244:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763931.196249:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763931.196252:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763931.196255:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763931.196257:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763931.196260:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763931.836207:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763931.836210:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763931.836214:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763931.836217:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763931.836223:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763931.836223:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763931.836227:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763931.836231:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763931.836233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763931.836239:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763931.836239:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763931.836244:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763931.836244:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763931.836248:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763931.836250:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763931.836253:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787763931.836255:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763931.836257:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763931.836262:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763931.836264:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763931.836265:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763931.836314:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763931.836319:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763931.836323:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763932.220278:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763932.220287:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763932.220290:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763932.220297:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763932.220301:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763932.220305:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.220309:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763932.220311:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.220314:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763932.220479:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763932.220486:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763932.220489:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763932.220493:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763932.220497:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763932.220505:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21646
00000400:00000200:0.0:1787763932.220513:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763932.220521:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763932.220525:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763932.220529:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763932.220534:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763932.220536:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763932.220540:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763932.220544:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763932.220546:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763932.220549:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763932.220553:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763932.220556:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763932.220558:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763932.220562:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763932.220564:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763932.220566:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763932.220568:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763932.220571:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763932.220575:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763932.220577:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763932.220580:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763932.220584:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763932.220587:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763932.220590:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763932.220593:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763932.220602:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787763932.220606:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763932.220610:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763932.220612:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763932.220623:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763932.220626:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763932.220631:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763932.220640:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787763932.220647:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763932.220657:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832674176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763932.220674:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763932.220679:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763932.220684:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832674176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763932.220714:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763932.220717:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763932.220721:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763932.220722:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763932.220724:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763932.220727:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763932.220730:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21646
00000400:00000200:0.0:1787763932.220734:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763932.220737:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763932.220739:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763932.220742:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763932.220744:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763932.220746:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763932.220748:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763932.220750:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763932.220751:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763932.220752:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763932.220755:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763932.220757:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763932.220759:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763932.220761:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763932.220763:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763932.220765:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763932.220767:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763932.220770:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763932.220773:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763932.220775:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763932.220777:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763932.220783:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763932.220786:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763932.220787:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763932.220792:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763932.220795:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763932.220797:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763932.220801:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000020:00000040:0.0:1787763932.220803:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763932.220805:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763932.220806:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763932.220808:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763932.220809:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763932.220811:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763932.220816:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763932.220820:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763932.220825:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832674304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763932.220832:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763932.220835:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763932.220839:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832674304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763932.220847:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763932.220849:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763932.220852:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763932.220854:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763932.220855:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763932.220857:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763932.220860:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21646
00000400:00000200:0.0:1787763932.220864:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763932.220867:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763932.220870:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763932.220872:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763932.220874:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763932.220877:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763932.220880:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763932.220882:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763932.220884:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763932.220885:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763932.220887:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763932.220889:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763932.220892:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763932.220896:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763932.220899:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763932.220901:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763932.220903:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763932.220905:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763932.220908:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763932.220910:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763932.220912:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763932.220916:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763932.220918:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763932.220921:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763932.220924:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763932.220927:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763932.220929:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763932.220934:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
00000100:00000001:3.0:1787763932.220934:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:0.0:1787763932.220941:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000001:3.0:1787763932.220941:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763932.220946:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
02000000:00000001:0.0:1787763932.220947:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763932.220951:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763932.220955:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:3.0:1787763932.220956:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832674176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763932.220957:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763932.220961:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763932.220968:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:3.0:1787763932.220972:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763932.220974:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763932.220975:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763932.220983:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00100000:0.0:1787763932.220986:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832674432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1787763932.220987:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763932.220994:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763932.221002:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832674176:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763932.221003:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763932.221009:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763932.221010:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763932.221016:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:0.0:1787763932.221019:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832674432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787763932.221020:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763932.221025:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763932.221031:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610c00.
00000100:00000001:0.0:1787763932.221036:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763932.221036:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763932.221041:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1787763932.221044:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:0.0:1787763932.221048:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000010:3.0:1787763932.221052:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000100:00080000:0.0:1787763932.221054:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763932.221060:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000010:3.0:1787763932.221061:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000001:0.0:1787763932.221063:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763932.221068:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763932.221072:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:3.0:1787763932.221073:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832674176, portal 25
00000100:00000001:3.0:1787763932.221077:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00080000:0.0:1787763932.221079:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21646
00000100:00000001:3.0:1787763932.221080:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
00000400:00000200:0.0:1787763932.221088:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:3.0:1787763932.221090:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832674176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787763932.221096:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763932.221103:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787763932.221109:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787763932.221110:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000040:3.0:1787763932.221116:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763932.221117:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:3.0:1787763932.221120:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000040:0.0:1787763932.221124:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000200:3.0:1787763932.221126:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832674176, offset 0
00000100:00000001:0.0:1787763932.221130:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763932.221161:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763932.221163:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763932.221165:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763932.221169:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763932.221171:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763932.221174:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000200:3.0:1787763932.221179:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787763932.221181:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763932.221186:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763932.221189:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:3.0:1787763932.221192:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763932.221194:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763932.221199:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000400:00000200:3.0:1787763932.221202:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00080000:0.0:1787763932.221207:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:3.0:1787763932.221210:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1787763932.221212:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763932.221216:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1787763932.221218:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
02000000:00000001:0.0:1787763932.221222:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1787763932.221225:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
02000000:00000001:0.0:1787763932.221228:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763932.221230:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:0.0:1787763932.221234:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1787763932.221238:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58502:58502:256:4294967295] 192.168.203.110@tcp LPNI seq info [58502:58502:8:4294967295]
00000008:00100000:0.0:1787763932.221240:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787763932.221246:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763932.221247:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763932.221251:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763932.221257:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000400:00000200:3.0:1787763932.221263:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000020:00000040:0.0:1787763932.221264:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763932.221272:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763932.221275:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787763932.221276:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763932.221279:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763932.221283:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000010:3.0:1787763932.221285:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
02000000:00000001:0.0:1787763932.221286:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763932.221295:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000800:00000200:3.0:1787763932.221296:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763932.221301:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787763932.221303:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000100:00100000:0.0:1787763932.221310:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832674560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1787763932.221314:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763932.221321:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000100:00000001:0.0:1787763932.221329:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000100:3.0:1787763932.221329:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763932.221334:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763932.221339:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763932.221343:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1787763932.221346:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763932.221349:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763932.221353:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000040:0.0:1787763932.221355:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832674560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.221359:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763932.221368:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876c000 x1874599832674176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763932.221372:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763932.221380:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21656)
00000100:00000001:3.0:1787763932.221384:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763932.221389:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832674176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763957 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.221397:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763932.221399:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763932.221405:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763932.221409:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763932.221412:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316320500
00000100:00000010:3.0:1787763932.221416:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40660.
00000100:00000001:3.0:1787763932.221423:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787763932.221430:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316320500
00000100:00000010:3.0:1787763932.221434:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40ea0.
00000100:00000001:3.0:1787763932.221437:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787763932.221440:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763932.221443:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763932.221445:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763932.221450:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763932/real 1787763932]  req@ffff9751f876c000 x1874599832674176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763932.221458:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763932.221459:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763932.221461:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763932.221463:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763932.221464:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763932.221466:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763932.221468:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832674176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763932.221474:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832674176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.221481:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763932.221484:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.221487:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763932.221489:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763932.221494:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832674176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.221501:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763932.221506:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832674176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.221513:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763932.221516:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.221518:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763932.221519:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.221522:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763932.221524:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763932.221527:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763932.221531:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763932.221534:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763932.221539:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832674176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763932.221549:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832674176:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787763932.221554:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763932.221557:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cb40 x1874599832674304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.221565:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763932.221566:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763932.221569:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763932.221571:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763932.221573:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763932.221577:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832674304:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763932.221589:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763932.221592:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763932.221594:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763932.221596:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763932.221601:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787763932.221603:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763932.221607:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763932.221609:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763932.221613:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763932.221617:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832674304, portal 10
00000100:00000001:3.0:1787763932.221619:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763932.221621:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049520960 : -115096660030656 : ffff9751f876cb40)
00000100:00000040:3.0:1787763932.221628:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cb40 x1874599832674304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.221635:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763932.221638:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763932.221641:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763932.221643:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832674304, offset 0
00000400:00000200:3.0:1787763932.221648:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763932.221652:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763932.221657:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763932.221660:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763932.221662:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763932.221665:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763932.221668:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763932.221671:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58503:58503:256:4294967295] 192.168.203.110@tcp LPNI seq info [58503:58503:8:4294967295]
00000400:00000200:3.0:1787763932.221675:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763932.221682:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763932.221687:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763932.221691:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763932.221707:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763932.221711:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1787763932.221716:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763932.221719:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000100:3.0:1787763932.221723:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763932.221724:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763932.221728:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763932.221731:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763932.221735:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763932.221737:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763932.221740:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763932.221743:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cb40 x1874599832674304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.221749:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763932.221751:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cb40 x1874599832674304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763957 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.221755:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763932.221756:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763932.221759:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763932.221762:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787763932.221764:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763932.221766:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316319264
00000100:00000001:3.0:1787763932.221768:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316320500
00000100:00000001:3.0:1787763932.221769:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316319264
00000100:00000001:3.0:1787763932.221771:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316320500
00000100:00000001:3.0:1787763932.221772:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763932.221774:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763932.221775:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763932.221778:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763932/real 1787763932]  req@ffff9751f876cb40 x1874599832674304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763932.221783:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763932.221784:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763932.221785:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763932.221787:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763932.221788:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763932.221789:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763932.221791:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cb40 x1874599832674304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763932.221799:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cb40 x1874599832674304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.221806:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763932.221807:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.221809:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763932.221811:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763932.221814:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cb40 x1874599832674304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.221821:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763932.221824:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cb40 x1874599832674304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.221831:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763932.221833:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.221835:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763932.221836:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.221838:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763932.221851:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763932.221854:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763932.221858:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763932.221860:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763932.221863:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cb40 x1874599832674304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763932.221871:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832674304:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763932.221875:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.221877:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763932.221881:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832674176/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.221887:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763932.221889:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763932.221892:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610c00.
02000000:00000001:3.0:1787763932.221894:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763932.221897:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763932.221898:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787763932.221901:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763932.221903:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016c00.
02000000:00000001:3.0:1787763932.221907:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763932.221909:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763932.221910:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:3.0:1787763932.221914:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763932.221915:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.221916:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763932.221919:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cb40 x1874599832674304/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.221925:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763932.221926:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763932.221928:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787763932.221929:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763932.221931:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763932.221932:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763932.221933:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763932.221935:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017800.
02000000:00000001:3.0:1787763932.221938:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763932.221939:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763932.221940:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cb40.
00000100:00000001:3.0:1787763932.221942:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763932.221943:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.221946:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.221953:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763932.221955:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763932.221957:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763932.221959:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763932.221961:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763932.221964:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876da40 x1874599832674432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.221971:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763932.221973:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763932.221975:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763932.221977:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763932.221979:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763932.221982:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832674432:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763932.221985:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763932.221987:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763932.221989:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763932.221991:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763932.221994:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787763932.221996:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763932.222000:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763932.222002:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763932.222005:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763932.222008:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832674432, portal 10
00000100:00000001:3.0:1787763932.222010:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763932.222011:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049524800 : -115096660026816 : ffff9751f876da40)
00000100:00000040:3.0:1787763932.222017:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876da40 x1874599832674432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.222025:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763932.222027:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763932.222030:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763932.222033:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832674432, offset 0
00000400:00000200:3.0:1787763932.222037:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763932.222041:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763932.222045:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763932.222049:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763932.222052:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763932.222054:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763932.222057:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763932.222059:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58504:58504:256:4294967295] 192.168.203.110@tcp LPNI seq info [58504:58504:8:4294967295]
00000400:00000200:3.0:1787763932.222063:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763932.222070:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763932.222074:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763932.222078:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763932.222083:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763932.222086:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1787763932.222090:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763932.222093:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000100:3.0:1787763932.222096:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763932.222097:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763932.222100:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763932.222102:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763932.222104:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763932.222106:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763932.222107:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763932.222110:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876da40 x1874599832674432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.222116:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763932.222118:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876da40 x1874599832674432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763957 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.222122:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763932.222123:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763932.222126:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763932.222128:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787763932.222130:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763932.222132:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316319252
00000100:00000001:3.0:1787763932.222153:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316320500
00000100:00000001:3.0:1787763932.222156:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316319253
00000100:00000001:3.0:1787763932.222157:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316320500
00000100:00000001:3.0:1787763932.222159:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763932.222160:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763932.222162:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763932.222164:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763932/real 1787763932]  req@ffff9751f876da40 x1874599832674432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763932.222170:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763932.222171:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763932.222172:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763932.222174:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763932.222175:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763932.222176:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763932.222178:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876da40 x1874599832674432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763932.222184:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876da40 x1874599832674432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.222188:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763932.222189:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.222190:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763932.222191:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763932.222193:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876da40 x1874599832674432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.222197:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763932.222200:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876da40 x1874599832674432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.222204:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763932.222206:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.222207:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763932.222209:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.222210:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763932.222211:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763932.222213:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763932.222215:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763932.222217:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763932.222219:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876da40 x1874599832674432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763932.222224:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832674432:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763932.222226:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763932.222228:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876de00 x1874599832674560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.222232:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763932.222233:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763932.222234:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763932.222236:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763932.222237:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763932.222239:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832674560:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763932.222242:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763932.222244:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763932.222245:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763932.222247:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763932.222250:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610c00.
02000000:00000001:3.0:1787763932.222253:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763932.222256:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763932.222259:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763932.222262:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763932.222265:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832674560, portal 4
00000100:00000001:3.0:1787763932.222267:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763932.222269:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049525760 : -115096660025856 : ffff9751f876de00)
00000100:00000040:3.0:1787763932.222275:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876de00 x1874599832674560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.222282:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763932.222285:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763932.222287:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763932.222290:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832674560, offset 0
00000400:00000200:3.0:1787763932.222294:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763932.222299:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763932.222303:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763932.222306:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763932.222308:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763932.222310:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763932.222312:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763932.222314:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58505:58505:256:4294967295] 192.168.203.110@tcp LPNI seq info [58505:58505:8:4294967295]
00000400:00000200:3.0:1787763932.222317:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763932.222322:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763932.222326:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763932.222330:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763932.222334:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763932.222337:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000200:3.1:1787763932.222341:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763932.222344:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 23 secs later)
00000800:00000100:3.0:1787763932.222348:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763932.222350:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763932.222353:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763932.222356:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763932.222359:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763932.222361:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763932.222364:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763932.222368:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876de00 x1874599832674560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.222376:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763932.222379:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876de00 x1874599832674560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763957 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.222392:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763932.222394:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763932.222398:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763932.222401:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787763932.222404:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763932.222405:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316319270
00000100:00000001:3.0:1787763932.222407:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316320500
00000100:00000001:3.0:1787763932.222408:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316319270
00000100:00000001:3.0:1787763932.222410:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316320500
00000100:00000001:3.0:1787763932.222411:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763932.222413:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763932.222414:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763932.222417:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763932/real 1787763932]  req@ffff9751f876de00 x1874599832674560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763932.222422:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763932.222423:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763932.222424:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763932.222426:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763932.222427:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763932.222429:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763932.222430:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876de00 x1874599832674560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763932.222436:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876de00 x1874599832674560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.222440:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763932.222441:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.222442:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763932.222443:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763932.222446:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876de00 x1874599832674560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.222450:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763932.222453:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876de00 x1874599832674560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.222457:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763932.222458:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.222459:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763932.222460:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.222461:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763932.222462:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763932.222464:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763932.222466:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763932.222467:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763932.222469:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876de00 x1874599832674560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763932.222477:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832674560:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763932.222480:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.222482:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763932.222486:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876da40 x1874599832674432/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.222492:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763932.222494:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763932.222496:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787763932.222499:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763932.222500:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763932.222502:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763932.222504:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763932.222506:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
02000000:00000001:3.0:1787763932.222509:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763932.222510:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763932.222511:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876da40.
00000100:00000001:3.0:1787763932.222515:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763932.222516:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.222518:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763932.222522:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876de00 x1874599832674560/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763957 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763932.222528:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763932.222529:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763932.222530:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610c00.
02000000:00000001:3.0:1787763932.222532:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763932.222533:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763932.222533:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787763932.222535:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763932.222536:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:3.0:1787763932.222537:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763932.222538:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763932.222539:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876de00.
00000100:00000001:3.0:1787763932.222542:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763932.222542:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.222544:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763932.222546:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763932.222547:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763932.222548:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763932.222549:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763932.860237:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763932.860248:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763932.860258:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763932.860259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763932.860263:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763932.860269:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763932.860272:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763932.860274:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763932.860277:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763932.860282:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787763932.860289:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763932.860293:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763932.860297:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763932.860300:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763932.860305:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763932.860337:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763932.860344:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763932.860348:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763933.244278:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763933.244288:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763933.244290:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763933.244297:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763933.244308:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763933.244315:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763933.244321:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763933.244326:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763933.244331:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763933.500195:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316320500, at 4316320512
00000100:00000010:3.0:1787763933.500213:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40ea0.
00000100:00000040:2.0:1787763933.500234:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316320500, at 4316320512
00000100:00000010:2.0:1787763933.500242:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40660.
00000100:00000001:2.0:1787763933.884320:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763933.884320:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763933.884330:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763933.884331:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763933.884335:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763933.884335:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763933.884341:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763933.884345:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763933.884346:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763933.884351:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763933.884353:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763933.884355:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763933.884360:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763933.884361:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763933.884365:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763933.884370:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763933.884370:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763933.884374:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763933.884377:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763933.884383:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763933.884395:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763933.884431:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763933.884437:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763933.884441:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763934.268259:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763934.268269:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763934.268271:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763934.268277:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763934.268281:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763934.268285:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763934.268287:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763934.268290:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763934.268292:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763934.908240:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763934.908247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763934.908248:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763934.908251:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763934.908252:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763934.908256:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763934.908260:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763934.908261:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763934.908265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763934.908266:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763934.908271:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763934.908275:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763934.908275:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763934.908280:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763934.908280:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763934.908357:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763934.908360:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763934.908363:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763935.292205:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763935.292211:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763935.292213:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763935.292217:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763935.292227:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763935.292230:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763935.292233:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763935.292235:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763935.292242:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763935.932239:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763935.932244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763935.932248:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763935.932249:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763935.932251:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763935.932255:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763935.932256:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763935.932258:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763935.932259:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763935.932262:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763935.932263:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763935.932269:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763935.932273:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763935.932276:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763935.932279:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763935.932315:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763935.932321:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763935.932325:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763936.316234:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763936.316242:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763936.316246:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763936.316252:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763936.316257:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763936.316261:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763936.316265:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763936.316268:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763936.316271:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763936.956235:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763936.956236:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763936.956241:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763936.956244:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763936.956247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763936.956249:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763936.956252:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763936.956254:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00100000:2.0:1787763936.956255:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763936.956258:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763936.956261:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763936.956261:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763936.956265:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763936.956267:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763936.956268:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763936.956273:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763936.956274:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763936.956277:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763936.956279:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763936.956279:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763936.956286:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763936.956342:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763936.956347:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763936.956351:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787763937.340246:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763937.340255:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787763937.340257:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787763937.340260:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:1.0:1787763937.340261:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763937.340267:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763937.340268:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763937.340272:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763937.340274:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21651
00000400:00000200:1.0:1787763937.340283:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00100000:2.0:1787763937.340289:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000400:00000200:1.0:1787763937.340290:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763937.340294:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787763937.340297:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1787763937.340299:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763937.340303:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763937.340304:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763937.340309:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763937.340310:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763937.340314:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763937.340315:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:1.0:1787763937.340318:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763937.340321:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000100:00000001:2.0:1787763937.340321:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787763937.340326:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763937.340330:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787763937.340332:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787763937.340334:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787763937.340337:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763937.340339:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763937.340341:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763937.340343:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763937.340345:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787763937.340348:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763937.340350:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763937.340353:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763937.340355:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763937.340357:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763937.340359:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787763937.340361:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763937.340368:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7931680.
00000020:00000040:1.0:1787763937.340370:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787763937.340373:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763937.340375:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763937.340377:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763937.340379:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763937.340382:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763937.340390:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca000.
00000100:00000001:1.0:1787763937.340396:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763937.340406:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7931680 x1874599832674688/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787763937.340417:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763937.340421:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763937.340426:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d4255d6] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7931680 x1874599832674688/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787763937.340454:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763937.340457:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763937.340460:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787763937.340461:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763937.340462:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763937.340464:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763937.340466:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21651
00000400:00000200:1.0:1787763937.340469:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763937.340472:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763937.340474:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763937.340477:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763937.340479:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763937.340481:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763937.340483:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763937.340485:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763937.340486:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787763937.340488:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787763937.340489:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:1.0:1787763937.340490:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787763937.340494:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787763937.340496:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:3.0:1787763937.340496:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1787763937.340501:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763937.340501:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1787763937.340505:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763937.340509:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763937.340510:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:3.0:1787763937.340512:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7931680 x1874599832674688/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787763937.340513:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787763937.340518:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763937.340523:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763937.340527:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763937.340533:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763937.340537:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763937.340540:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763937.340544:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787763937.340550:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763937.340551:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763937.340552:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763937.340556:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787763937.340557:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930b40.
00000020:00000040:1.0:1787763937.340562:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763937.340564:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763937.340564:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787763937.340568:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763937.340570:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763937.340570:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763937.340573:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763937.340577:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763937.340577:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763937.340585:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8400.
00000100:00100000:3.0:1787763937.340586:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832674688:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787763937.340589:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763937.340595:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00100000:1.0:1787763937.340599:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930b40 x1874599832674816/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1787763937.340601:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763937.340606:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763937.340608:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763937.340611:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763937.340611:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:1.0:1787763937.340617:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022d7f91a] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930b40 x1874599832674816/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1787763937.340621:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610c00.
00000100:00000001:1.0:1787763937.340624:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763937.340626:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787763937.340627:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763937.340632:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787763937.340633:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787763937.340634:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1787763937.340638:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787763937.340639:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763937.340643:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21651
00000400:00000200:1.0:1787763937.340647:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763937.340648:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000200:1.0:1787763937.340649:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763937.340654:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000010:3.0:1787763937.340659:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000040:1.0:1787763937.340678:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763937.340684:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:3.0:1787763937.340686:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832674688, portal 25
00000100:00000040:1.0:1787763937.340687:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763937.340691:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787763937.340693:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000020:00000001:1.0:1787763937.340695:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763937.340698:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000100:00000001:3.0:1787763937.340698:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497727104 : -115097211824512 : ffff9751d7931680)
00000020:00000001:1.0:1787763937.340701:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763937.340703:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787763937.340705:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787763937.340707:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787763937.340712:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763937.340714:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763937.340714:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7931680 x1874599832674688/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787763937.340718:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763937.340722:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763937.340726:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787763937.340732:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763937.340737:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763937.340739:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:1.0:1787763937.340740:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787763937.340744:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1787763937.340748:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
02000000:00000001:1.0:1787763937.340749:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763937.340753:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763937.340754:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000010:3.0:1787763937.340755:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000008:00000001:1.0:1787763937.340757:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763937.340761:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787763937.340762:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832674688, offset 0
00000100:00000010:1.0:1787763937.340764:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930000.
00000020:00000040:1.0:1787763937.340768:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787763937.340770:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763937.340771:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763937.340775:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787763937.340776:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1787763937.340776:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
02000000:00000001:1.0:1787763937.340779:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763937.340784:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9400.
00000100:00000001:1.0:1787763937.340786:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763937.340788:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00100000:1.0:1787763937.340793:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930000 x1874599832674944/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787763937.340801:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:1.0:1787763937.340803:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763937.340807:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763937.340811:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:1.0:1787763937.340813:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6d90274] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930000 x1874599832674944/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787763937.340819:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:1.0:1787763937.340825:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763937.340828:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000040:1.0:1787763937.340830:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787763937.340835:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:3.0:1787763937.340836:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00080000:1.0:1787763937.340838:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787763937.340842:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787763937.340845:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1787763937.340845:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58506:58506:256:4294967295] 192.168.203.110@tcp LPNI seq info [58506:58506:8:4294967295]
00000100:00080000:1.0:1787763937.340848:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787763937.340854:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787763937.340859:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21651
00000400:00000200:3.0:1787763937.340859:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763937.340867:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787763937.340871:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787763937.340875:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787763937.340881:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:3.0:1787763937.340882:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:1.0:1787763937.340884:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763937.340889:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763937.340891:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787763937.340894:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787763937.340897:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000800:00000200:3.0:1787763937.340897:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000020:00000001:1.0:1787763937.340901:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787763937.340904:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787763937.340905:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787763937.340906:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000800:00000010:3.0:1787763937.340908:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000100:00000040:1.0:1787763937.340910:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763937.340913:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787763937.340915:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787763937.340916:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787763937.340920:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000800:00000200:3.0:1787763937.340923:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00080000:1.0:1787763937.340925:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787763937.340929:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787763937.340931:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:3.0:1787763937.340931:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
02000000:00000001:1.0:1787763937.340933:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787763937.340939:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763937.340941:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787763937.340943:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1787763937.340947:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763937.340947:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787763937.340951:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787763937.340954:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000100:00000010:1.0:1787763937.340956:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933480.
00000020:00000040:1.0:1787763937.340961:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787763937.340964:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787763937.340966:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1787763937.340966:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787763937.340970:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000010:3.0:1787763937.340971:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
02000000:00000001:1.0:1787763937.340972:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787763937.340975:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787763937.340979:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cac00.
00000400:00000200:3.0:1787763937.340981:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:1.0:1787763937.340982:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763937.340988:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933480 x1874599832675072/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787763937.340994:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:1.0:1787763937.340998:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787763937.341000:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787763937.341004:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3c2c33c] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933480 x1874599832675072/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787763937.341009:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000100:00000001:1.0:1787763937.341010:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787763937.341015:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000040:1.0:1787763937.341016:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21662)
00000100:00000001:3.0:1787763937.341023:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763937.341030:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7931680 x1874599832674688/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341043:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763937.341048:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7931680 x1874599832674688/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763962 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341056:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763937.341058:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763937.341065:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763937.341069:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763937.341072:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316325619
00000100:00000010:3.0:1787763937.341078:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40ea0.
00000100:00000001:3.0:1787763937.341085:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787763937.341092:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316325619
00000100:00000010:3.0:1787763937.341095:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40ba0.
00000100:00000001:3.0:1787763937.341098:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787763937.341101:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763937.341105:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763937.341107:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763937.341112:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763937/real 1787763937]  req@ffff9751d7931680 x1874599832674688/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763937.341122:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763937.341124:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763937.341126:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763937.341128:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763937.341130:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763937.341155:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763937.341160:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7931680 x1874599832674688/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763937.341170:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7931680 x1874599832674688/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341179:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763937.341182:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763937.341185:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763937.341187:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763937.341193:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7931680 x1874599832674688/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341200:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763937.341205:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7931680 x1874599832674688/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341213:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763937.341216:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763937.341218:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763937.341220:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763937.341222:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763937.341224:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763937.341228:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763937.341232:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763937.341235:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763937.341241:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7931680 x1874599832674688/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763937.341249:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832674688:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787763937.341254:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763937.341256:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763937.341260:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7931680 x1874599832674688/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341266:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763937.341269:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763937.341272:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610c00.
02000000:00000001:3.0:1787763937.341274:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763937.341276:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763937.341278:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787763937.341281:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763937.341283:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca000.
02000000:00000001:3.0:1787763937.341287:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763937.341289:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763937.341291:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7931680.
00000100:00000001:3.0:1787763937.341294:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763937.341296:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763937.341298:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763937.341305:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763937.341307:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763937.341309:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763937.341311:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763937.341312:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763937.341317:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930b40 x1874599832674816/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341324:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763937.341325:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763937.341327:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763937.341329:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763937.341332:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763937.341335:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832674816:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763937.341339:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763937.341341:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763937.341343:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763937.341345:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763937.341348:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610c00.
02000000:00000001:3.0:1787763937.341350:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763937.341354:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763937.341357:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763937.341361:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763937.341364:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832674816, portal 10
00000100:00000001:3.0:1787763937.341367:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763937.341368:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497724224 : -115097211827392 : ffff9751d7930b40)
00000100:00000040:3.0:1787763937.341375:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930b40 x1874599832674816/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341382:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763937.341385:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763937.341387:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763937.341390:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832674816, offset 0
00000400:00000200:3.0:1787763937.341395:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763937.341399:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763937.341403:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763937.341406:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763937.341409:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763937.341411:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763937.341414:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763937.341417:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58507:58507:256:4294967295] 192.168.203.110@tcp LPNI seq info [58507:58507:8:4294967295]
00000400:00000200:3.0:1787763937.341421:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763937.341428:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763937.341433:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763937.341437:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763937.341441:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763937.341445:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787763937.341449:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763937.341453:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787763937.341457:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763937.341459:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763937.341462:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763937.341466:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763937.341469:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763937.341471:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763937.341474:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763937.341477:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930b40 x1874599832674816/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341486:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763937.341490:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930b40 x1874599832674816/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763962 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341497:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763937.341498:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763937.341502:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763937.341506:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787763937.341510:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763937.341512:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316324384
00000100:00000001:3.0:1787763937.341515:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316325619
00000100:00000001:3.0:1787763937.341517:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316324384
00000100:00000001:3.0:1787763937.341520:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316325619
00000100:00000001:3.0:1787763937.341522:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763937.341524:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763937.341526:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763937.341530:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763937/real 1787763937]  req@ffff9751d7930b40 x1874599832674816/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763937.341546:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763937.341548:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763937.341550:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763937.341552:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763937.341554:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763937.341556:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763937.341560:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930b40 x1874599832674816/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763937.341569:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930b40 x1874599832674816/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341577:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763937.341579:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763937.341581:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763937.341582:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763937.341587:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930b40 x1874599832674816/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341595:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763937.341599:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930b40 x1874599832674816/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341606:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763937.341609:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763937.341611:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763937.341612:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763937.341614:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763937.341626:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763937.341629:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763937.341632:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763937.341635:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763937.341639:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930b40 x1874599832674816/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763937.341648:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832674816:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763937.341652:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763937.341656:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930000 x1874599832674944/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341677:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763937.341679:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763937.341681:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763937.341683:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763937.341685:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763937.341688:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832674944:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763937.341692:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763937.341694:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763937.341695:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763937.341697:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763937.341701:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787763937.341703:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763937.341707:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763937.341710:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763937.341713:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763937.341716:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832674944, portal 10
00000100:00000001:3.0:1787763937.341718:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763937.341719:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497721344 : -115097211830272 : ffff9751d7930000)
00000100:00000040:3.0:1787763937.341725:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930000 x1874599832674944/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341733:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763937.341735:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763937.341738:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763937.341741:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832674944, offset 0
00000400:00000200:3.0:1787763937.341746:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763937.341749:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763937.341753:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763937.341756:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763937.341759:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763937.341762:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763937.341764:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763937.341768:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58508:58508:256:4294967295] 192.168.203.110@tcp LPNI seq info [58508:58508:8:4294967295]
00000400:00000200:3.0:1787763937.341773:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763937.341779:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763937.341784:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763937.341787:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763937.341791:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763937.341794:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787763937.341799:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763937.341802:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787763937.341806:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763937.341808:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763937.341811:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763937.341815:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763937.341818:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763937.341820:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763937.341823:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763937.341827:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930000 x1874599832674944/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341836:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763937.341840:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930000 x1874599832674944/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763962 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341847:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763937.341848:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763937.341851:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763937.341854:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787763937.341857:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763937.341860:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316324372
00000100:00000001:3.0:1787763937.341863:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316325619
00000100:00000001:3.0:1787763937.341866:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316324372
00000100:00000001:3.0:1787763937.341868:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316325619
00000100:00000001:3.0:1787763937.341870:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763937.341873:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763937.341875:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763937.341880:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763937/real 1787763937]  req@ffff9751d7930000 x1874599832674944/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763937.341889:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763937.341890:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763937.341892:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763937.341895:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763937.341897:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763937.341899:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763937.341903:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930000 x1874599832674944/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763937.341912:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930000 x1874599832674944/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341921:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763937.341923:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763937.341924:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763937.341926:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763937.341930:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930000 x1874599832674944/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341938:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763937.341942:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930000 x1874599832674944/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341950:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763937.341954:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763937.341955:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763937.341957:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763937.341959:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763937.341961:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763937.341964:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763937.341968:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763937.341970:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763937.341974:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930000 x1874599832674944/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763937.341983:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832674944:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763937.341988:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763937.341991:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933480 x1874599832675072/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.341999:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763937.342001:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763937.342003:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763937.342005:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763937.342007:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763937.342010:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832675072:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763937.342014:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763937.342016:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763937.342018:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763937.342020:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763937.342024:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787763937.342026:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763937.342030:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763937.342032:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763937.342036:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763937.342039:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832675072, portal 4
00000100:00000001:3.0:1787763937.342041:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763937.342043:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497734784 : -115097211816832 : ffff9751d7933480)
00000100:00000040:3.0:1787763937.342048:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933480 x1874599832675072/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.342056:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763937.342058:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763937.342061:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763937.342063:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832675072, offset 0
00000400:00000200:3.0:1787763937.342067:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763937.342071:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763937.342074:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763937.342078:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763937.342080:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763937.342083:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763937.342085:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763937.342089:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58509:58509:256:4294967295] 192.168.203.110@tcp LPNI seq info [58509:58509:8:4294967295]
00000400:00000200:3.0:1787763937.342094:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763937.342100:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763937.342105:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763937.342108:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763937.342113:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763937.342116:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787763937.342119:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763937.342122:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787763937.342127:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763937.342129:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763937.342149:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763937.342153:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763937.342156:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763937.342158:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763937.342161:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763937.342166:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933480 x1874599832675072/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.342181:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763937.342184:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933480 x1874599832675072/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763962 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.342192:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763937.342193:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763937.342196:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763937.342199:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787763937.342202:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763937.342204:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316324390
00000100:00000001:3.0:1787763937.342207:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316325619
00000100:00000001:3.0:1787763937.342209:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316324390
00000100:00000001:3.0:1787763937.342212:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316325619
00000100:00000001:3.0:1787763937.342214:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763937.342216:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763937.342218:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763937.342222:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763937/real 1787763937]  req@ffff9751d7933480 x1874599832675072/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763937.342230:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763937.342232:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763937.342234:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763937.342237:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763937.342239:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763937.342241:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763937.342245:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933480 x1874599832675072/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763937.342253:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933480 x1874599832675072/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.342261:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763937.342263:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763937.342265:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763937.342266:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763937.342270:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933480 x1874599832675072/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.342278:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763937.342282:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933480 x1874599832675072/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.342289:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763937.342291:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763937.342293:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763937.342294:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763937.342296:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763937.342298:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763937.342301:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763937.342304:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763937.342307:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763937.342311:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933480 x1874599832675072/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763937.342320:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832675072:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763937.342325:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763937.342327:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763937.342330:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930b40 x1874599832674816/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.342337:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763937.342338:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763937.342341:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610c00.
02000000:00000001:3.0:1787763937.342343:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763937.342345:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763937.342347:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763937.342349:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763937.342351:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8400.
02000000:00000001:3.0:1787763937.342353:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763937.342355:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763937.342357:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930b40.
00000100:00000001:3.0:1787763937.342360:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763937.342361:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763937.342363:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763937.342367:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930000 x1874599832674944/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.342375:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763937.342376:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763937.342378:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787763937.342380:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763937.342382:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763937.342383:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763937.342386:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763937.342387:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9400.
02000000:00000001:3.0:1787763937.342390:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763937.342391:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763937.342393:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930000.
00000100:00000001:3.0:1787763937.342395:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763937.342396:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763937.342398:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763937.342402:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933480 x1874599832675072/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763962 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763937.342410:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763937.342411:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763937.342413:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787763937.342416:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763937.342417:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763937.342418:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787763937.342420:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763937.342421:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cac00.
02000000:00000001:3.0:1787763937.342423:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763937.342425:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763937.342426:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933480.
00000100:00000001:3.0:1787763937.342430:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763937.342432:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763937.342434:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763937.342438:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763937.342440:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763937.342442:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763937.342444:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763937.980251:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763937.980254:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763937.980258:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763937.980259:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763937.980261:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763937.980266:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763937.980268:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763937.980270:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763937.980271:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763937.980273:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763937.980276:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763937.980278:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763937.980279:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763937.980282:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763937.980283:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763937.980336:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763937.980340:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763937.980343:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763938.364242:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763938.364250:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763938.364253:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763938.364258:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763938.364262:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763938.364265:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763938.364268:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763938.364271:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763938.364274:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763938.620197:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316325619, at 4316325632
00000100:00000010:3.0:1787763938.620211:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40ba0.
00000100:00000040:2.0:1787763938.620215:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316325619, at 4316325632
00000100:00000010:2.0:1787763938.620223:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40ea0.
00010000:00000001:3.0:1787763939.004246:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763939.004248:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763939.004254:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763939.004256:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763939.004261:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763939.004263:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763939.004267:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763939.004267:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763939.004274:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763939.004274:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763939.004278:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763939.004279:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763939.004284:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763939.004287:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763939.004290:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763939.004330:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763939.004334:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763939.004337:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763939.388211:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763939.388218:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763939.388219:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763939.388224:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763939.388228:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763939.388230:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763939.388232:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763939.388234:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763939.388236:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763940.028221:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763940.028230:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763940.028233:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763940.028238:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787763940.028240:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763940.028242:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763940.028248:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763940.028248:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763940.028252:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763940.028255:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763940.028256:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763940.028258:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763940.028269:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763940.028273:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763940.028277:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763940.028313:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763940.028317:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763940.028319:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763940.412244:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763940.412253:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763940.412257:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763940.412263:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763940.412267:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763940.412271:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763940.412274:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763940.412277:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763940.412280:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763941.052199:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763941.052206:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763941.052208:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763941.052213:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763941.052214:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763941.052216:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763941.052218:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763941.052221:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763941.052224:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763941.052226:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763941.052229:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763941.052233:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763941.052235:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763941.052239:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763941.052240:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763941.052329:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763941.052334:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763941.052339:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763941.436202:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763941.436209:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763941.436211:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763941.436216:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763941.436225:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763941.436228:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763941.436230:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763941.436232:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763941.436234:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763942.076255:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763942.076259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763942.076264:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763942.076265:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763942.076268:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763942.076272:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763942.076273:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763942.076278:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763942.076279:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763942.076283:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763942.076284:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787763942.076286:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763942.076289:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787763942.076290:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763942.076293:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787763942.076295:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763942.076299:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763942.076304:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763942.076307:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763942.076309:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763942.076312:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763942.076403:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763942.076410:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763942.076416:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763942.460293:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1787763942.460293:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763942.460304:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:3.0:1787763942.460305:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787763942.460307:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763942.460308:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:2.0:1787763942.460315:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:3.0:1787763942.460316:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763942.460322:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763942.460323:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763942.460327:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763942.460330:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:3.0:1787763942.460333:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21657
00000100:00000001:2.0:1787763942.460334:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763942.460340:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763942.460346:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763942.460354:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763942.460357:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763942.460361:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763942.460364:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763942.460366:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763942.460368:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763942.460371:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763942.460373:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787763942.460375:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763942.460379:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787763942.460380:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787763942.460382:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787763942.460385:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763942.460387:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763942.460388:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763942.460389:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763942.460393:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787763942.460396:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763942.460398:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763942.460401:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763942.460404:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763942.460407:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763942.460408:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787763942.460411:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763942.460421:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bde00.
00000020:00000040:3.0:1787763942.460423:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787763942.460427:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763942.460429:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763942.460432:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763942.460434:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763942.460439:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763942.460446:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612000.
00000100:00000001:3.0:1787763942.460453:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763942.460462:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bde00 x1874599832675200/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787763942.460472:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787763942.460476:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763942.460480:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000941163c2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bde00 x1874599832675200/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763942.460499:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763942.460502:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787763942.460504:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787763942.460505:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763942.460507:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763942.460509:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787763942.460512:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21657
00000400:00000200:3.0:1787763942.460515:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763942.460519:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763942.460521:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763942.460523:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763942.460524:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763942.460526:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763942.460529:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1787763942.460530:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763942.460534:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1787763942.460534:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763942.460538:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763942.460540:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:3.0:1787763942.460542:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:2.0:1787763942.460544:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bde00 x1874599832675200/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787763942.460545:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:3.0:1787763942.460549:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:3.0:1787763942.460553:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:3.0:1787763942.460562:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763942.460566:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763942.460567:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763942.460570:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763942.460572:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763942.460577:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:2.0:1787763942.460578:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:3.0:1787763942.460580:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:2.0:1787763942.460584:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:3.0:1787763942.460586:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:2.0:1787763942.460591:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763942.460593:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763942.460598:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:2.0:1787763942.460600:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832675200:192.168.203.110@tcp:250:kworker.0
02000000:00000001:3.0:1787763942.460606:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763942.460608:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763942.460612:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763942.460615:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1787763942.460617:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1787763942.460620:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1787763942.460635:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1787763942.460637:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:3.0:1787763942.460642:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763942.460644:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763942.460645:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
00000100:00000010:3.0:1787763942.460648:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bfc00.
02000000:00000001:2.0:1787763942.460649:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1787763942.460654:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787763942.460658:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:2.0:1787763942.460660:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:3.0:1787763942.460661:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763942.460665:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:2.0:1787763942.460667:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
02000000:00000001:3.0:1787763942.460668:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763942.460672:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763942.460678:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
02000000:00000010:3.0:1787763942.460678:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610800.
00000100:00000001:3.0:1787763942.460685:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763942.460689:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832675200, portal 25
00000100:00100000:3.0:1787763942.460691:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bfc00 x1874599832675328/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763942.460693:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763942.460695:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049853440 : -115096659698176 : ffff9751f87bde00)
00000100:00000001:3.0:1787763942.460704:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:2.0:1787763942.460706:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bde00 x1874599832675200/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763942.460707:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763942.460716:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003c1f6b31] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bfc00 x1874599832675328/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.460721:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763942.460729:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787763942.460731:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763942.460733:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00080000:3.0:1787763942.460735:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:2.0:1787763942.460737:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832675200, offset 0
00000100:00080000:3.0:1787763942.460739:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1787763942.460741:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763942.460743:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1787763942.460748:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787763942.460749:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787763942.460754:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21657
00000400:00000200:2.0:1787763942.460760:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763942.460769:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763942.460776:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763942.460781:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763942.460786:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763942.460790:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763942.460795:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58510:58510:256:4294967295] 192.168.203.110@tcp LPNI seq info [58510:58510:8:4294967295]
00000400:00000200:2.0:1787763942.460800:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763942.460811:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763942.460819:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763942.460824:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000010:2.0:1787763942.460828:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000400:00000200:3.0:1787763942.460833:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763942.460837:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000200:2.0:1787763942.460838:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:3.0:1787763942.460844:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:2.0:1787763942.460845:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:3.0:1787763942.460850:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.1:1787763942.460855:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:3.0:1787763942.460857:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000200:2.1:1787763942.460859:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:3.0:1787763942.460864:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000100:2.0:1787763942.460869:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000020:00000001:3.0:1787763942.460871:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763942.460874:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000800:00000010:2.0:1787763942.460875:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000020:00000001:3.0:1787763942.460877:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763942.460882:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:3.0:1787763942.460884:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:2.0:1787763942.460887:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:3.0:1787763942.460888:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:3.0:1787763942.460897:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:2.0:1787763942.460898:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1787763942.460904:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1787763942.460908:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000100:00000040:3.0:1787763942.460909:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763942.460913:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000010:2.0:1787763942.460915:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000020:00000040:3.0:1787763942.460916:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:3.0:1787763942.460922:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763942.460923:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:3.0:1787763942.460924:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763942.460927:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:2.0:1787763942.460931:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bde00 x1874599832675200/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787763942.460932:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763942.460937:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763942.460941:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787763942.460945:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787763942.460948:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763942.460949:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1787763942.460950:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763942.460955:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bde00 x1874599832675200/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763967 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1787763942.460956:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc3c0.
00000020:00000040:3.0:1787763942.460961:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787763942.460966:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763942.460968:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763942.460972:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763942.460972:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763942.460975:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787763942.460976:0:257575:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:3.0:1787763942.460978:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763942.460982:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610c00.
00000100:00000001:2.0:1787763942.460984:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763942.460986:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763942.460991:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00100000:3.0:1787763942.460994:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc3c0 x1874599832675456/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763942.460996:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316330739
00000100:00000001:3.0:1787763942.461005:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000010:2.0:1787763942.461006:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00000001:3.0:1787763942.461008:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763942.461013:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000060bf8470] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc3c0 x1874599832675456/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.461016:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787763942.461023:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763942.461026:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763942.461027:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316330739
00000100:00000040:3.0:1787763942.461031:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000010:2.0:1787763942.461033:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb71e0.
00000100:00080000:3.0:1787763942.461037:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763942.461038:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00080000:3.0:1787763942.461044:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787763942.461045:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763942.461046:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787763942.461048:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763942.461049:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763942.461053:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:3.0:1787763942.461054:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787763942.461059:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21657
00000400:00000200:3.0:1787763942.461062:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763942.461064:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763942.461066:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00100000:2.0:1787763942.461068:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763942/real 1787763942]  req@ffff9751f87bde00 x1874599832675200/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787763942.461071:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763942.461075:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763942.461080:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763942.461084:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763942.461088:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763942.461090:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:3.0:1787763942.461093:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763942.461096:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:3.0:1787763942.461100:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000001:2.0:1787763942.461103:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000001:3.0:1787763942.461104:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000001:2.0:1787763942.461108:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:3.0:1787763942.461110:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000010:2.0:1787763942.461111:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000100:00000001:3.0:1787763942.461113:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1787763942.461116:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000100:00000040:3.0:1787763942.461118:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000010:2.0:1787763942.461120:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:3.0:1787763942.461122:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763942.461124:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000040:3.0:1787763942.461126:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000200:2.0:1787763942.461131:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bde00 x1874599832675200/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763942.461159:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bde00 x1874599832675200/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.461164:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763942.461167:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1787763942.461167:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763942.461173:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763942.461174:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763942.461176:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787763942.461178:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:2.0:1787763942.461183:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bde00 x1874599832675200/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787763942.461185:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763942.461193:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787763942.461193:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763942.461199:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:2.0:1787763942.461202:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bde00 x1874599832675200/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:3.0:1787763942.461205:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:2.0:1787763942.461214:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000008:00000001:3.0:1787763942.461215:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763942.461219:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763942.461221:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763942.461222:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763942.461225:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763942.461228:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000010:3.0:1787763942.461229:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc780.
00000100:00000001:2.0:1787763942.461230:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000040:3.0:1787763942.461235:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00080000:2.0:1787763942.461236:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000001:3.0:1787763942.461243:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00080000:2.0:1787763942.461244:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
02000000:00000001:3.0:1787763942.461245:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763942.461248:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:3.0:1787763942.461249:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763942.461253:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:2.0:1787763942.461255:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bde00 x1874599832675200/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787763942.461256:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763942.461262:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611c00.
00000100:00000001:3.0:1787763942.461267:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763942.461272:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832675200:192.168.203.110@tcp:250:kworker.0
00000100:00100000:3.0:1787763942.461276:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc780 x1874599832675584/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763942.461278:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763942.461283:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763942.461289:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bde00 x1874599832675200/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763942.461290:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787763942.461295:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763942.461303:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
00000100:00000040:3.0:1787763942.461303:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7d56184] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc780 x1874599832675584/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787763942.461307:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763942.461310:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
00000100:00000001:3.0:1787763942.461314:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763942.461315:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763942.461319:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763942.461320:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000100:00000040:3.0:1787763942.461321:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21667)
00000020:00000001:2.0:1787763942.461324:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763942.461327:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612000.
02000000:00000001:2.0:1787763942.461331:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763942.461333:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763942.461335:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bde00.
00000100:00000001:2.0:1787763942.461338:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763942.461340:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763942.461341:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763942.461348:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763942.461349:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763942.461351:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763942.461352:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763942.461353:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763942.461357:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bfc00 x1874599832675328/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.461362:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763942.461363:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763942.461365:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763942.461366:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763942.461368:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763942.461369:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832675328:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763942.461372:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763942.461374:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763942.461375:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763942.461376:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763942.461379:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763942.461380:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763942.461383:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763942.461385:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000400:00000010:2.0:1787763942.461388:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000200:2.0:1787763942.461390:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832675328, portal 10
00000100:00000001:2.0:1787763942.461391:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763942.461392:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049861120 : -115096659690496 : ffff9751f87bfc00)
00000100:00000040:2.0:1787763942.461398:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bfc00 x1874599832675328/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.461402:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763942.461404:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763942.461406:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000200:2.0:1787763942.461407:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832675328, offset 0
00000400:00000200:2.0:1787763942.461410:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763942.461413:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763942.461415:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763942.461417:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763942.461419:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763942.461421:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763942.461423:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763942.461425:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58511:58511:256:4294967295] 192.168.203.110@tcp LPNI seq info [58511:58511:8:4294967295]
00000400:00000200:2.0:1787763942.461428:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763942.461433:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763942.461435:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763942.461438:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000800:00000200:2.0:1787763942.461440:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763942.461443:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787763942.461446:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763942.461448:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1787763942.461450:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763942.461451:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000400:00000200:2.0:1787763942.461453:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763942.461456:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763942.461458:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:2.0:1787763942.461459:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:2.0:1787763942.461460:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763942.461463:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bfc00 x1874599832675328/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.461468:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763942.461470:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bfc00 x1874599832675328/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763967 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.461474:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763942.461475:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763942.461477:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763942.461480:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787763942.461482:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763942.461484:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316329504
00000100:00000001:2.0:1787763942.461486:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316330739
00000100:00000001:2.0:1787763942.461488:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316329504
00000100:00000001:2.0:1787763942.461489:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316330739
00000100:00000001:2.0:1787763942.461491:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763942.461492:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763942.461493:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763942.461495:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763942/real 1787763942]  req@ffff9751f87bfc00 x1874599832675328/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763942.461500:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763942.461501:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763942.461502:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763942.461504:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763942.461505:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:2.0:1787763942.461507:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763942.461509:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bfc00 x1874599832675328/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763942.461514:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bfc00 x1874599832675328/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.461518:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763942.461519:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763942.461520:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763942.461521:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763942.461523:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bfc00 x1874599832675328/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.461527:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763942.461529:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bfc00 x1874599832675328/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.461533:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763942.461535:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763942.461536:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763942.461537:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763942.461538:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763942.461550:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763942.461553:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763942.461557:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763942.461559:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763942.461563:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bfc00 x1874599832675328/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763942.461572:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832675328:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763942.461577:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763942.461581:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc3c0 x1874599832675456/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.461588:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763942.461590:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763942.461592:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763942.461594:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763942.461597:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763942.461601:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832675456:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763942.461603:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763942.461605:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763942.461606:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763942.461607:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763942.461610:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763942.461612:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763942.461614:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763942.461616:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000400:00000010:2.0:1787763942.461618:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000200:2.0:1787763942.461621:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832675456, portal 10
00000100:00000001:2.0:1787763942.461622:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763942.461623:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049846720 : -115096659704896 : ffff9751f87bc3c0)
00000100:00000040:2.0:1787763942.461644:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc3c0 x1874599832675456/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.461649:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763942.461650:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763942.461652:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000200:2.0:1787763942.461654:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832675456, offset 0
00000400:00000200:2.0:1787763942.461657:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763942.461660:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763942.461663:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763942.461665:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763942.461667:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763942.461669:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763942.461671:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763942.461674:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58512:58512:256:4294967295] 192.168.203.110@tcp LPNI seq info [58512:58512:8:4294967295]
00000400:00000200:2.0:1787763942.461676:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763942.461680:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763942.461683:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763942.461686:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000800:00000200:2.0:1787763942.461688:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763942.461690:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787763942.461693:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763942.461695:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1787763942.461698:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763942.461699:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000400:00000200:2.0:1787763942.461701:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763942.461704:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763942.461706:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:2.0:1787763942.461708:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:2.0:1787763942.461709:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763942.461712:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc3c0 x1874599832675456/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.461717:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763942.461719:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc3c0 x1874599832675456/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763967 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.461724:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763942.461725:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763942.461727:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763942.461729:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787763942.461731:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763942.461733:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316329492
00000100:00000001:2.0:1787763942.461735:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316330739
00000100:00000001:2.0:1787763942.461737:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316329492
00000100:00000001:2.0:1787763942.461739:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316330739
00000100:00000001:2.0:1787763942.461741:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763942.461742:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763942.461743:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763942.461746:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763942/real 1787763942]  req@ffff9751f87bc3c0 x1874599832675456/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763942.461751:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763942.461752:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763942.461753:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763942.461755:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763942.461756:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:2.0:1787763942.461757:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763942.461759:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc3c0 x1874599832675456/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763942.461765:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc3c0 x1874599832675456/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.461772:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763942.461774:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763942.461776:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763942.461777:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763942.461781:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc3c0 x1874599832675456/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.461788:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763942.461793:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc3c0 x1874599832675456/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.461801:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763942.461804:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763942.461806:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763942.461807:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763942.461810:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763942.461812:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763942.461814:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763942.461818:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763942.461821:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763942.461826:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc3c0 x1874599832675456/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763942.461832:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832675456:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763942.461835:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763942.461837:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc780 x1874599832675584/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.461842:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763942.461843:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763942.461844:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763942.461846:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763942.461847:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763942.461849:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832675584:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763942.461851:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763942.461853:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763942.461854:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763942.461855:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763942.461858:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763942.461860:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763942.461863:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763942.461864:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000400:00000010:2.0:1787763942.461868:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000200:2.0:1787763942.461871:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832675584, portal 4
00000100:00000001:2.0:1787763942.461873:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763942.461875:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049847680 : -115096659703936 : ffff9751f87bc780)
00000100:00000040:2.0:1787763942.461881:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc780 x1874599832675584/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.461889:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763942.461891:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763942.461894:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000200:2.0:1787763942.461896:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832675584, offset 0
00000400:00000200:2.0:1787763942.461901:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763942.461905:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763942.461909:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763942.461912:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763942.461915:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763942.461917:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763942.461920:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763942.461923:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58513:58513:256:4294967295] 192.168.203.110@tcp LPNI seq info [58513:58513:8:4294967295]
00000400:00000200:2.0:1787763942.461926:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763942.461930:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763942.461933:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763942.461936:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000800:00000200:2.0:1787763942.461938:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763942.461940:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787763942.461943:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763942.461945:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1787763942.461947:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763942.461949:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000400:00000200:2.0:1787763942.461951:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763942.461953:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763942.461955:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:2.0:1787763942.461956:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:2.0:1787763942.461958:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763942.461961:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc780 x1874599832675584/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.461966:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763942.461968:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc780 x1874599832675584/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763967 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.461977:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763942.461978:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763942.461981:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763942.461983:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787763942.461985:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763942.461987:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316329509
00000100:00000001:2.0:1787763942.461989:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316330739
00000100:00000001:2.0:1787763942.461990:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316329509
00000100:00000001:2.0:1787763942.461992:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316330739
00000100:00000001:2.0:1787763942.461993:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763942.461994:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763942.461996:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763942.461998:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763942/real 1787763942]  req@ffff9751f87bc780 x1874599832675584/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763942.462003:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763942.462004:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763942.462005:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763942.462006:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763942.462008:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:2.0:1787763942.462009:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763942.462011:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc780 x1874599832675584/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763942.462017:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc780 x1874599832675584/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.462020:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763942.462021:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763942.462022:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763942.462023:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763942.462025:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc780 x1874599832675584/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.462030:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763942.462033:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc780 x1874599832675584/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.462037:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763942.462039:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763942.462040:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763942.462041:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763942.462042:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763942.462043:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763942.462044:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763942.462046:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763942.462047:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763942.462050:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc780 x1874599832675584/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763942.462054:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832675584:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763942.462056:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763942.462058:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763942.462060:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bfc00 x1874599832675328/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.462065:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763942.462066:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763942.462067:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763942.462069:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763942.462070:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763942.462071:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763942.462072:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763942.462073:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610800.
02000000:00000001:2.0:1787763942.462075:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763942.462077:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763942.462078:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bfc00.
00000100:00000001:2.0:1787763942.462080:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763942.462081:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763942.462082:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763942.462084:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc3c0 x1874599832675456/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.462089:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763942.462090:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763942.462091:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763942.462092:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763942.462093:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763942.462093:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763942.462094:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763942.462095:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610c00.
02000000:00000001:2.0:1787763942.462096:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763942.462097:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763942.462098:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc3c0.
00000100:00000001:2.0:1787763942.462099:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763942.462100:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763942.462101:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763942.462102:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc780 x1874599832675584/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763967 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763942.462106:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763942.462107:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763942.462108:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763942.462109:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763942.462109:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763942.462110:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787763942.462111:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763942.462112:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611c00.
02000000:00000001:2.0:1787763942.462113:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763942.462114:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763942.462115:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc780.
00000100:00000001:2.0:1787763942.462116:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763942.462117:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763942.462119:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763942.462121:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763942.462122:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763942.462123:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763942.462125:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763943.100235:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763943.100242:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763943.100248:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763943.100250:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763943.100254:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763943.100255:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763943.100263:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763943.100269:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763943.100273:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763943.100278:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763943.100283:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763943.100287:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763943.100291:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763943.100295:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763943.100299:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763943.100325:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763943.100330:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763943.100334:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763943.484267:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763943.484277:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763943.484280:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763943.484288:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763943.484292:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763943.484296:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763943.484299:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763943.484303:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763943.484307:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763943.740227:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316330739, at 4316330752
00000100:00000010:3.0:1787763943.740246:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb71e0.
00000100:00000040:2.0:1787763943.740259:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316330739, at 4316330752
00000100:00000010:2.0:1787763943.740266:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00000100:00000001:1.0:1787763944.124256:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763944.124261:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763944.124270:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763944.124271:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:1.0:1787763944.124275:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763944.124276:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00100000:1.0:1787763944.124283:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763944.124284:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763944.124292:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763944.124293:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:1.0:1787763944.124299:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763944.124299:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763944.124306:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763944.124307:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763944.124312:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763944.124313:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763944.124318:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763944.124323:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763944.124326:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763944.124334:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763944.124338:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763944.124389:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763944.124393:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763944.124396:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763944.508190:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763944.508201:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763944.508204:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763944.508211:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763944.508216:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763944.508221:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763944.508225:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763944.508228:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763944.508233:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763945.148202:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763945.148211:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763945.148213:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763945.148219:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763945.148223:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763945.148227:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763945.148230:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763945.148233:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763945.148236:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763945.148237:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763945.148242:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763945.148247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763945.148248:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763945.148252:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763945.148254:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763945.148275:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763945.148279:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763945.148283:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763945.532244:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763945.532253:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763945.532255:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763945.532261:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763945.532264:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763945.532268:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763945.532271:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763945.532274:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763945.532276:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763946.172277:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763946.172281:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763946.172283:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763946.172291:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763946.172292:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763946.172303:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763946.172305:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763946.172312:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763946.172314:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763946.172319:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763946.172320:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763946.172322:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763946.172324:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763946.172327:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763946.172329:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763946.172376:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763946.172381:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763946.172384:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763946.556238:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763946.556244:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763946.556246:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763946.556252:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763946.556256:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763946.556259:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763946.556261:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763946.556264:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763946.556266:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763947.196211:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763947.196219:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763947.196221:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763947.196225:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787763947.196226:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763947.196230:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763947.196232:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763947.196236:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763947.196237:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763947.196240:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:1.0:1787763947.196241:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763947.196244:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:1.0:1787763947.196245:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763947.196248:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763947.196249:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763947.196252:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763947.196255:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763947.196256:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763947.196258:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763947.196261:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763947.196263:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763947.196317:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763947.196320:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763947.196322:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763947.580255:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763947.580263:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763947.580265:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763947.580269:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763947.580273:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763947.580276:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763947.580279:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763947.580281:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763947.580283:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763947.580298:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787763947.580307:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787763947.580310:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763947.580315:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763947.580322:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787763947.580330:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21662
00000400:00000200:3.0:1787763947.580340:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763947.580349:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763947.580352:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763947.580355:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763947.580357:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763947.580359:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763947.580361:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763947.580365:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763947.580367:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787763947.580370:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763947.580374:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787763947.580376:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787763947.580378:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787763947.580381:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763947.580383:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763947.580384:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763947.580386:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763947.580389:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787763947.580392:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763947.580394:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763947.580397:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763947.580401:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763947.580403:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763947.580405:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787763947.580407:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763947.580418:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bcf00.
00000020:00000040:3.0:1787763947.580421:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787763947.580424:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763947.580425:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763947.580429:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763947.580432:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763947.580436:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763947.580445:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610400.
00000100:00000001:3.0:1787763947.580452:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763947.580461:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bcf00 x1874599832675712/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787763947.580470:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787763947.580474:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763947.580478:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ebc1bc5] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bcf00 x1874599832675712/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763947.580498:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763947.580500:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787763947.580503:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787763947.580504:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763947.580506:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763947.580508:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787763947.580510:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21662
00000400:00000200:3.0:1787763947.580513:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763947.580515:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763947.580517:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763947.580521:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763947.580522:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763947.580525:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1787763947.580525:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763947.580529:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763947.580530:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763947.580535:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1787763947.580535:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763947.580546:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:3.0:1787763947.580550:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763947.580552:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:3.0:1787763947.580558:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:2.0:1787763947.580559:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bcf00 x1874599832675712/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787763947.580563:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:3.0:1787763947.580579:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763947.580604:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763947.580605:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763947.580606:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763947.580611:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:2.0:1787763947.580613:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1787763947.580615:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763947.580619:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:2.0:1787763947.580620:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:3.0:1787763947.580624:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:2.0:1787763947.580631:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763947.580631:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763947.580636:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:2.0:1787763947.580645:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832675712:192.168.203.110@tcp:250:kworker.0
02000000:00000001:3.0:1787763947.580645:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763947.580654:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763947.580661:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787763947.580663:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:3.0:1787763947.580666:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787763947.580670:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763947.580671:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1787763947.580672:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763947.580676:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763947.580680:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc780.
02000000:00000001:2.0:1787763947.580681:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:3.0:1787763947.580686:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787763947.580689:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763947.580689:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763947.580691:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763947.580693:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:2.0:1787763947.580694:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:3.0:1787763947.580696:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763947.580697:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763947.580706:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611800.
00000400:00000200:2.0:1787763947.580715:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763947.580717:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763947.580720:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000100:00100000:3.0:1787763947.580725:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc780 x1874599832675840/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:2.0:1787763947.580730:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000001:3.0:1787763947.580741:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:2.0:1787763947.580742:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832675712, portal 25
00000100:00000001:2.0:1787763947.580747:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763947.580749:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763947.580750:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049849600 : -115096659702016 : ffff9751f87bcf00)
00000100:00000040:2.0:1787763947.580766:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bcf00 x1874599832675712/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763972 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787763947.580769:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7d56184] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc780 x1874599832675840/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.580792:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763947.580805:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763947.580809:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000001:3.0:1787763947.580811:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763947.580814:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832675712, offset 0
00000100:00080000:3.0:1787763947.580816:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787763947.580821:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:2.0:1787763947.580822:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787763947.580826:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763947.580829:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763947.580832:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:2.0:1787763947.580835:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:3.0:1787763947.580836:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21662
00000400:00000200:2.0:1787763947.580845:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763947.580851:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763947.580854:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763947.580857:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763947.580860:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763947.580863:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58514:58514:256:4294967295] 192.168.203.110@tcp LPNI seq info [58514:58514:8:4294967295]
00000400:00000200:2.0:1787763947.580866:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763947.580881:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000400:00000200:3.0:1787763947.580889:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:2.0:1787763947.580897:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763947.580900:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763947.580904:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000010:2.0:1787763947.580907:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000100:00000040:3.0:1787763947.580911:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763947.580916:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1787763947.580923:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:3.0:1787763947.580923:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763947.580933:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:2.0:1787763947.580934:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:3.0:1787763947.580942:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763947.580945:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:3.0:1787763947.580951:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:2.1:1787763947.580953:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:3.0:1787763947.580958:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000200:2.1:1787763947.580959:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:3.0:1787763947.580965:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:3.0:1787763947.580969:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000100:2.0:1787763947.580974:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763947.580981:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000100:00000040:3.0:1787763947.580983:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763947.580987:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1787763947.580989:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1787763947.580992:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763947.580996:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787763947.581004:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000020:00000040:3.0:1787763947.581004:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:3.0:1787763947.581013:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:2.0:1787763947.581015:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000100:00000001:3.0:1787763947.581017:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763947.581021:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
02000000:00000001:3.0:1787763947.581021:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787763947.581029:0:257575:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:3.0:1787763947.581029:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763947.581033:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763947.581036:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bcf00 x1874599832675712/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763972 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763947.581037:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787763947.581040:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787763947.581051:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763947.581060:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763947.581070:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000010:3.0:1787763947.581071:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc3c0.
00000100:00000040:2.0:1787763947.581079:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bcf00 x1874599832675712/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763972 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787763947.581079:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787763947.581092:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763947.581094:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763947.581096:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763947.581097:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763947.581098:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763947.581106:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613c00.
00000100:00000001:2.0:1787763947.581114:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763947.581116:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763947.581118:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763947.581125:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763947.581125:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc3c0 x1874599832675968/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787763947.581176:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763947.581177:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763947.581184:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316335860
00000100:00000001:3.0:1787763947.581185:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763947.581203:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00000040:3.0:1787763947.581203:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000060bf8470] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc3c0 x1874599832675968/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.581215:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787763947.581215:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763947.581225:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1787763947.581228:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1787763947.581229:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316335860
00000100:00080000:3.0:1787763947.581236:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000010:2.0:1787763947.581237:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7060.
00000100:00000001:2.0:1787763947.581246:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00080000:3.0:1787763947.581246:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787763947.581255:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787763947.581257:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:3.0:1787763947.581259:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763947.581262:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763947.581263:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763947.581267:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00080000:3.0:1787763947.581270:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21662
00000100:00100000:2.0:1787763947.581278:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763947/real 1787763947]  req@ffff9751f87bcf00 x1874599832675712/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787763947.581278:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763947.581286:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763947.581295:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763947.581311:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000001:2.0:1787763947.581312:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763947.581318:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:3.0:1787763947.581319:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1787763947.581321:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000100:00000040:3.0:1787763947.581327:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787763947.581330:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000100:00000001:3.0:1787763947.581331:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000010:2.0:1787763947.581340:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000020:00000001:3.0:1787763947.581340:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763947.581347:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000100:00000001:2.0:1787763947.581348:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000001:3.0:1787763947.581352:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000200:2.0:1787763947.581358:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bcf00 x1874599832675712/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787763947.581358:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:3.0:1787763947.581367:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:3.0:1787763947.581375:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:3.0:1787763947.581389:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763947.581394:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:2.0:1787763947.581396:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bcf00 x1874599832675712/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787763947.581402:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763947.581410:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763947.581418:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:3.0:1787763947.581431:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763947.581432:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763947.581439:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763947.581439:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763947.581450:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:3.0:1787763947.581451:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787763947.581462:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787763947.581464:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763947.581466:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763947.581470:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bcf00 x1874599832675712/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763947.581472:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787763947.581480:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1787763947.581491:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763947.581492:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763947.581507:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763947.581513:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bcf00 x1874599832675712/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1787763947.581515:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bfc00.
00000020:00000040:3.0:1787763947.581526:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:3.0:1787763947.581536:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1787763947.581537:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:3.0:1787763947.581542:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763947.581544:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763947.581548:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1787763947.581549:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:3.0:1787763947.581559:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787763947.581560:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787763947.581567:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763947.581568:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763947.581577:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000010:3.0:1787763947.581578:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613800.
00000100:00000001:3.0:1787763947.581604:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763947.581605:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763947.581615:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00100000:3.0:1787763947.581619:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bfc00 x1874599832676096/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763947.581622:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:3.0:1787763947.581635:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787763947.581639:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763947.581643:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bcf00 x1874599832675712/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787763947.581645:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003c1f6b31] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bfc00 x1874599832676096/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763947.581703:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763947.581709:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832675712:192.168.203.110@tcp:250:kworker.0
00000100:00000040:3.0:1787763947.581709:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21672)
00000100:00000001:2.0:1787763947.581722:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763947.581725:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763947.581729:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bcf00 x1874599832675712/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.581735:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763947.581737:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763947.581740:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763947.581743:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763947.581746:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763947.581749:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787763947.581751:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763947.581754:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610400.
02000000:00000001:2.0:1787763947.581758:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763947.581759:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763947.581761:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bcf00.
00000100:00000001:2.0:1787763947.581763:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763947.581764:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763947.581766:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763947.581772:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763947.581773:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763947.581775:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763947.581776:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763947.581778:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763947.581780:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc780 x1874599832675840/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.581790:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763947.581791:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763947.581793:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763947.581795:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763947.581797:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763947.581799:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832675840:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763947.581802:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763947.581804:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763947.581806:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763947.581808:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763947.581811:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763947.581813:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763947.581818:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763947.581821:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000400:00000010:2.0:1787763947.581824:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000200:2.0:1787763947.581828:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832675840, portal 10
00000100:00000001:2.0:1787763947.581829:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763947.581831:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049847680 : -115096659703936 : ffff9751f87bc780)
00000100:00000040:2.0:1787763947.581837:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc780 x1874599832675840/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763972 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.581842:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763947.581844:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763947.581847:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000200:2.0:1787763947.581849:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832675840, offset 0
00000400:00000200:2.0:1787763947.581854:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763947.581858:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763947.581862:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763947.581866:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763947.581868:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763947.581871:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763947.581874:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763947.581877:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58515:58515:256:4294967295] 192.168.203.110@tcp LPNI seq info [58515:58515:8:4294967295]
00000400:00000200:2.0:1787763947.581880:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763947.581887:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763947.581892:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763947.581895:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000800:00000200:2.0:1787763947.581898:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763947.581900:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787763947.581904:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763947.581906:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1787763947.581910:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763947.581912:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000400:00000200:2.0:1787763947.581916:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763947.581919:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763947.581926:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:2.0:1787763947.581928:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:2.0:1787763947.581933:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763947.581939:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc780 x1874599832675840/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763972 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.581947:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763947.581953:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc780 x1874599832675840/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763972 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.581964:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763947.581967:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763947.581972:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763947.581977:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787763947.581980:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763947.581982:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316334624
00000100:00000001:2.0:1787763947.581985:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316335860
00000100:00000001:2.0:1787763947.581987:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316334624
00000100:00000001:2.0:1787763947.581990:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316335860
00000100:00000001:2.0:1787763947.581993:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763947.581995:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763947.581997:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763947.582001:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763947/real 1787763947]  req@ffff9751f87bc780 x1874599832675840/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763947.582007:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763947.582008:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763947.582010:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763947.582013:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763947.582015:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:2.0:1787763947.582020:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763947.582025:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc780 x1874599832675840/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763947.582033:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc780 x1874599832675840/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.582040:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763947.582043:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763947.582044:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763947.582046:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763947.582048:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc780 x1874599832675840/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.582056:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763947.582063:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc780 x1874599832675840/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.582068:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763947.582071:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763947.582072:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763947.582073:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763947.582075:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763947.582093:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763947.582097:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763947.582099:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763947.582102:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763947.582105:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc780 x1874599832675840/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763947.582112:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832675840:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763947.582117:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763947.582121:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc3c0 x1874599832675968/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.582126:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763947.582128:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763947.582130:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763947.582166:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763947.582169:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763947.582172:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832675968:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763947.582176:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763947.582178:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763947.582180:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763947.582181:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763947.582184:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763947.582186:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763947.582189:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763947.582191:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000400:00000010:2.0:1787763947.582194:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000200:2.0:1787763947.582196:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832675968, portal 10
00000100:00000001:2.0:1787763947.582198:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763947.582200:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049846720 : -115096659704896 : ffff9751f87bc3c0)
00000100:00000040:2.0:1787763947.582205:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc3c0 x1874599832675968/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763972 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.582211:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763947.582214:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763947.582218:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000200:2.0:1787763947.582221:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832675968, offset 0
00000400:00000200:2.0:1787763947.582226:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763947.582230:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763947.582233:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763947.582240:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763947.582243:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763947.582246:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763947.582248:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763947.582251:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58516:58516:256:4294967295] 192.168.203.110@tcp LPNI seq info [58516:58516:8:4294967295]
00000400:00000200:2.0:1787763947.582256:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763947.582267:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763947.582270:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763947.582272:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000800:00000200:2.0:1787763947.582275:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763947.582277:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787763947.582280:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763947.582282:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1787763947.582285:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763947.582287:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000400:00000200:2.0:1787763947.582289:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763947.582292:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763947.582294:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:2.0:1787763947.582295:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:2.0:1787763947.582297:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763947.582300:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc3c0 x1874599832675968/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763972 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.582306:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763947.582308:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc3c0 x1874599832675968/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763972 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.582315:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763947.582317:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763947.582320:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763947.582322:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787763947.582325:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763947.582327:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316334613
00000100:00000001:2.0:1787763947.582329:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316335860
00000100:00000001:2.0:1787763947.582331:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316334613
00000100:00000001:2.0:1787763947.582333:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316335860
00000100:00000001:2.0:1787763947.582335:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763947.582336:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763947.582338:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763947.582344:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763947/real 1787763947]  req@ffff9751f87bc3c0 x1874599832675968/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763947.582353:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763947.582355:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763947.582357:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763947.582363:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763947.582365:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:2.0:1787763947.582371:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763947.582377:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc3c0 x1874599832675968/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763947.582384:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc3c0 x1874599832675968/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.582389:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763947.582390:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763947.582391:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763947.582392:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763947.582394:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc3c0 x1874599832675968/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.582399:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763947.582401:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc3c0 x1874599832675968/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.582405:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763947.582407:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763947.582408:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763947.582409:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763947.582410:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763947.582412:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763947.582413:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763947.582416:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763947.582418:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763947.582420:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc3c0 x1874599832675968/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763947.582425:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832675968:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763947.582428:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763947.582430:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bfc00 x1874599832676096/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.582435:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763947.582437:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763947.582438:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763947.582441:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763947.582442:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763947.582444:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832676096:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763947.582446:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763947.582448:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763947.582449:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763947.582450:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763947.582453:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763947.582454:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763947.582456:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763947.582458:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000400:00000010:2.0:1787763947.582460:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000200:2.0:1787763947.582463:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832676096, portal 4
00000100:00000001:2.0:1787763947.582464:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763947.582465:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049861120 : -115096659690496 : ffff9751f87bfc00)
00000100:00000040:2.0:1787763947.582469:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bfc00 x1874599832676096/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763972 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.582474:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763947.582476:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763947.582477:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000200:2.0:1787763947.582479:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832676096, offset 0
00000400:00000200:2.0:1787763947.582481:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763947.582483:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763947.582486:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763947.582487:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763947.582489:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763947.582491:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763947.582493:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763947.582495:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58517:58517:256:4294967295] 192.168.203.110@tcp LPNI seq info [58517:58517:8:4294967295]
00000400:00000200:2.0:1787763947.582498:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763947.582503:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763947.582506:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763947.582508:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000800:00000200:2.0:1787763947.582510:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763947.582512:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787763947.582514:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763947.582516:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1787763947.582519:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763947.582521:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000400:00000200:2.0:1787763947.582523:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763947.582525:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763947.582528:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:2.0:1787763947.582529:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:2.0:1787763947.582530:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763947.582533:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bfc00 x1874599832676096/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763972 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.582538:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763947.582540:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bfc00 x1874599832676096/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763972 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.582545:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763947.582546:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763947.582548:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763947.582549:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787763947.582551:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763947.582552:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316334630
00000100:00000001:2.0:1787763947.582554:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316335860
00000100:00000001:2.0:1787763947.582555:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316334630
00000100:00000001:2.0:1787763947.582557:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316335860
00000100:00000001:2.0:1787763947.582559:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763947.582561:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763947.582562:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763947.582564:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763947/real 1787763947]  req@ffff9751f87bfc00 x1874599832676096/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763947.582570:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763947.582570:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763947.582571:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763947.582573:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763947.582574:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:2.0:1787763947.582575:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763947.582577:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bfc00 x1874599832676096/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763947.582582:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bfc00 x1874599832676096/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.582587:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763947.582601:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763947.582605:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763947.582606:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763947.582609:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bfc00 x1874599832676096/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.582614:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763947.582617:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bfc00 x1874599832676096/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.582626:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763947.582628:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763947.582630:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763947.582632:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763947.582634:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763947.582635:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763947.582638:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763947.582643:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763947.582646:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763947.582651:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bfc00 x1874599832676096/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763947.582660:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832676096:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763947.582665:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763947.582667:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763947.582671:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc780 x1874599832675840/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.582677:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763947.582679:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763947.582680:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763947.582682:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763947.582684:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763947.582685:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763947.582686:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763947.582687:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611800.
02000000:00000001:2.0:1787763947.582690:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763947.582691:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763947.582692:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc780.
00000100:00000001:2.0:1787763947.582693:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763947.582694:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763947.582696:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763947.582698:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc3c0 x1874599832675968/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.582702:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763947.582703:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763947.582704:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763947.582706:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763947.582706:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763947.582707:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763947.582708:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763947.582710:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613c00.
02000000:00000001:2.0:1787763947.582712:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763947.582713:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763947.582714:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc3c0.
00000100:00000001:2.0:1787763947.582715:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763947.582716:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763947.582717:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763947.582719:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bfc00 x1874599832676096/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763972 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763947.582723:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763947.582724:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763947.582725:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763947.582726:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763947.582727:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763947.582727:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787763947.582729:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763947.582729:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613800.
02000000:00000001:2.0:1787763947.582731:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763947.582731:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763947.582732:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bfc00.
00000100:00000001:2.0:1787763947.582734:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763947.582735:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763947.582736:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763947.582738:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763947.582739:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763947.582740:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763947.582742:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763948.220293:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763948.220299:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763948.220305:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763948.220306:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763948.220310:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763948.220315:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787763948.220318:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763948.220319:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763948.220326:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763948.220329:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763948.220332:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763948.220332:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763948.220340:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763948.220344:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763948.220348:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763948.220397:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763948.220402:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763948.220405:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763948.604254:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763948.604261:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763948.604264:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763948.604271:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763948.604275:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763948.604279:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763948.604283:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763948.604286:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763948.604289:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763948.860231:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316335860, at 4316335872
00000100:00000010:3.0:1787763948.860251:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7060.
00000100:00000040:2.0:1787763948.860262:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316335860, at 4316335872
00000100:00000010:2.0:1787763948.860271:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00010000:00000001:3.0:1787763949.244246:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763949.244252:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763949.244256:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763949.244257:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763949.244261:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763949.244263:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763949.244283:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763949.244290:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763949.244293:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763949.244299:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763949.244303:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763949.244307:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763949.244309:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763949.244312:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763949.244317:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763949.244318:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763949.244324:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763949.244327:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763949.628276:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763949.628285:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763949.628288:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763949.628293:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763949.628298:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763949.628302:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763949.628305:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763949.628308:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763949.628311:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763950.268231:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763950.268231:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763950.268237:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763950.268238:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763950.268243:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763950.268245:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787763950.268247:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763950.268247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763950.268251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763950.268252:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763950.268253:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763950.268256:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763950.268260:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763950.268262:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763950.268264:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763950.268338:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763950.268344:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763950.268348:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763950.652226:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763950.652234:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763950.652236:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763950.652241:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763950.652247:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763950.652251:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763950.652254:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763950.652257:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763950.652260:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763951.292207:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763951.292215:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763951.292223:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763951.292225:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763951.292230:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763951.292235:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763951.292235:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:1.0:1787763951.292240:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763951.292240:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:1.0:1787763951.292243:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763951.292246:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763951.292247:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763951.292251:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:1.0:1787763951.292252:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763951.292257:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763951.292257:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763951.292264:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763951.292264:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763951.292267:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763951.292269:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763951.292270:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763951.292317:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763951.292320:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763951.292323:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763951.676242:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763951.676249:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763951.676251:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763951.676261:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763951.676265:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763951.676268:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763951.676270:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763951.676273:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763951.676275:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763952.316236:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763952.316240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763952.316244:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763952.316245:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763952.316250:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763952.316251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763952.316253:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763952.316257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787763952.316258:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763952.316258:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763952.316264:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763952.316268:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763952.316271:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763952.316274:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763952.316277:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763952.316313:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763952.316317:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763952.316320:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763952.700217:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1787763952.700217:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763952.700223:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:3.0:1787763952.700225:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787763952.700227:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763952.700228:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:2.0:1787763952.700232:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:3.0:1787763952.700233:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763952.700239:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763952.700240:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763952.700243:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.700248:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:3.0:1787763952.700248:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21667
00000100:00000001:2.0:1787763952.700252:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.700255:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763952.700259:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763952.700266:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763952.700268:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763952.700271:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763952.700274:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763952.700277:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763952.700278:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763952.700281:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763952.700282:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787763952.700284:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763952.700287:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787763952.700289:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787763952.700290:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787763952.700292:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763952.700294:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763952.700301:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763952.700302:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763952.700305:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787763952.700307:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763952.700309:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763952.700312:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763952.700314:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763952.700316:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763952.700317:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787763952.700319:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763952.700324:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bde00.
00000020:00000040:3.0:1787763952.700327:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787763952.700330:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763952.700331:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763952.700334:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763952.700337:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763952.700339:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763952.700346:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612400.
00000100:00000001:3.0:1787763952.700351:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763952.700359:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bde00 x1874599832676224/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787763952.700367:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787763952.700370:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763952.700373:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000941163c2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bde00 x1874599832676224/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763952.700387:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763952.700389:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787763952.700392:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787763952.700397:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763952.700399:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763952.700402:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787763952.700406:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21667
00000100:00000001:2.0:1787763952.700410:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:3.0:1787763952.700413:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763952.700415:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763952.700418:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000400:00000200:3.0:1787763952.700421:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763952.700426:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763952.700427:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bde00 x1874599832676224/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787763952.700435:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763952.700441:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763952.700442:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763952.700445:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763952.700446:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:2.0:1787763952.700449:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1787763952.700452:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:2.0:1787763952.700453:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763952.700459:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787763952.700459:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763952.700463:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00100000:2.0:1787763952.700465:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832676224:192.168.203.110@tcp:250:kworker.0
00000020:00000001:3.0:1787763952.700468:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1787763952.700472:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:3.0:1787763952.700473:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
02000000:00000001:2.0:1787763952.700476:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000001:3.0:1787763952.700478:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:2.0:1787763952.700479:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763952.700484:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000001:3.0:1787763952.700485:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000010:2.0:1787763952.700491:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
00000100:00000040:3.0:1787763952.700492:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:2.0:1787763952.700496:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763952.700497:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763952.700502:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787763952.700503:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763952.700509:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000010:2.0:1787763952.700510:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000400:00000010:2.0:1787763952.700517:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000020:00000040:3.0:1787763952.700522:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00000200:2.0:1787763952.700524:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832676224, portal 25
00000100:00000001:2.0:1787763952.700527:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763952.700530:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049853440 : -115096659698176 : ffff9751f87bde00)
00000100:00080000:3.0:1787763952.700533:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000040:2.0:1787763952.700539:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bde00 x1874599832676224/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763952.700539:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763952.700546:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787763952.700552:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:3.0:1787763952.700564:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1787763952.700570:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
02000000:00000001:3.0:1787763952.700570:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763952.700575:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000001:3.0:1787763952.700576:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000200:2.0:1787763952.700581:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832676224, offset 0
00000008:00100000:3.0:1787763952.700583:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787763952.700589:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763952.700591:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763952.700595:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000010:3.0:1787763952.700599:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87be1c0.
00000020:00000040:3.0:1787763952.700603:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000200:2.0:1787763952.700605:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:3.0:1787763952.700610:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:2.0:1787763952.700614:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
02000000:00000001:3.0:1787763952.700614:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763952.700618:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763952.700622:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1787763952.700623:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:3.0:1787763952.700627:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763952.700629:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763952.700634:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
02000000:00000010:3.0:1787763952.700634:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612c00.
00000400:00000200:2.0:1787763952.700637:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763952.700642:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58518:58518:256:4294967295] 192.168.203.110@tcp LPNI seq info [58518:58518:8:4294967295]
00000100:00000001:3.0:1787763952.700642:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763952.700648:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00100000:3.0:1787763952.700653:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87be1c0 x1874599832676352/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787763952.700660:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763952.700673:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787763952.700676:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000010:2.0:1787763952.700679:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000100:00000001:3.0:1787763952.700680:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787763952.700689:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:3.0:1787763952.700689:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e2011c6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87be1c0 x1874599832676352/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1787763952.700695:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787763952.700707:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787763952.700711:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787763952.700711:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1787763952.700717:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00080000:3.0:1787763952.700717:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:2.0:1787763952.700719:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000100:00080000:3.0:1787763952.700726:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:2.0:1787763952.700727:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1787763952.700729:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:2.0:1787763952.700733:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:3.0:1787763952.700734:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763952.700738:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:2.0:1787763952.700745:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000100:00080000:3.0:1787763952.700745:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21667
00000400:00000010:2.0:1787763952.700749:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:2.0:1787763952.700753:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000400:00000200:3.0:1787763952.700754:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000200:2.0:1787763952.700759:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bde00 x1874599832676224/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787763952.700762:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763952.700768:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787763952.700775:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763952.700775:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763952.700780:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763952.700785:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bde00 x1874599832676224/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763977 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787763952.700786:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763952.700792:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763952.700798:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1787763952.700799:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1787763952.700802:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000100:00000001:2.0:1787763952.700803:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:3.0:1787763952.700807:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1787763952.700809:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1787763952.700810:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000400:00000200:2.0:1787763952.700816:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:3.0:1787763952.700816:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000001:2.0:1787763952.700820:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316340979
00000020:00000001:3.0:1787763952.700823:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000010:2.0:1787763952.700826:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00000040:3.0:1787763952.700830:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763952.700834:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787763952.700834:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763952.700839:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763952.700844:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316340979
00000100:00000001:3.0:1787763952.700844:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763952.700850:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00000010:2.0:1787763952.700851:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7540.
00000100:00000001:2.0:1787763952.700856:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00080000:3.0:1787763952.700859:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763952.700860:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763952.700866:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763952.700867:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763952.700871:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000001:3.0:1787763952.700873:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:2.0:1787763952.700878:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763952/real 1787763952]  req@ffff9751f87bde00 x1874599832676224/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787763952.700879:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763952.700886:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:2.0:1787763952.700893:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:3.0:1787763952.700893:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000001:2.0:1787763952.700898:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000008:00100000:3.0:1787763952.700898:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000010:2.0:1787763952.700901:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000008:00000001:3.0:1787763952.700904:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763952.700905:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763952.700909:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:3.0:1787763952.700910:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763952.700912:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763952.700916:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bde00 x1874599832676224/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1787763952.700918:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc000.
00000020:00000040:3.0:1787763952.700924:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000200:2.0:1787763952.700929:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bde00 x1874599832676224/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787763952.700931:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763952.700934:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763952.700939:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1787763952.700940:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763952.700943:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787763952.700943:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787763952.700948:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:3.0:1787763952.700948:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763952.700950:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763952.700955:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bde00 x1874599832676224/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1787763952.700956:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612800.
00000100:00000001:2.0:1787763952.700964:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763952.700964:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763952.700979:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bde00 x1874599832676224/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763952.700980:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc000 x1874599832676480/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763952.700988:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763952.700991:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.700992:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763952.700994:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.700999:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763952.701003:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763952.701008:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00080000:2.0:1787763952.701010:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:3.0:1787763952.701013:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763952.701019:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000040:3.0:1787763952.701023:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3fc1c6c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc000 x1874599832676480/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701025:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763952.701038:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bde00 x1874599832676224/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763952.701040:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763952.701046:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1787763952.701055:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00100000:2.0:1787763952.701060:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832676224:192.168.203.110@tcp:250:kworker.0
00000100:00080000:3.0:1787763952.701061:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763952.701069:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1787763952.701070:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787763952.701073:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1787763952.701074:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763952.701077:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:2.0:1787763952.701078:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bde00 x1874599832676224/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763952.701082:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763952.701089:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
00000100:00080000:3.0:1787763952.701089:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21667
02000000:00000001:2.0:1787763952.701093:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763952.701098:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
00000400:00000200:3.0:1787763952.701098:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:2.0:1787763952.701104:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000400:00000200:3.0:1787763952.701106:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000020:00000001:2.0:1787763952.701107:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763952.701110:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787763952.701113:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
00000100:00080000:3.0:1787763952.701113:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
02000000:00000010:2.0:1787763952.701116:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612400.
00000100:00000040:3.0:1787763952.701120:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:2.0:1787763952.701122:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763952.701125:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000001:3.0:1787763952.701127:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000010:2.0:1787763952.701128:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bde00.
00000100:00000001:2.0:1787763952.701132:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763952.701151:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.701153:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.701158:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763952.701160:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763952.701161:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763952.701162:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763952.701165:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763952.701167:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763952.701168:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1787763952.701172:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000040:2.0:1787763952.701174:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87be1c0 x1874599832676352/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787763952.701176:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:3.0:1787763952.701180:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763952.701183:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:3.0:1787763952.701189:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:3.0:1787763952.701195:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:2.0:1787763952.701196:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763952.701200:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763952.701203:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:2.0:1787763952.701204:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763952.701208:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763952.701209:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1787763952.701210:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763952.701214:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00100000:2.0:1787763952.701215:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832676352:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763952.701220:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763952.701221:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763952.701225:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:3.0:1787763952.701226:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
02000000:00000001:2.0:1787763952.701227:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763952.701232:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763952.701235:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
00000100:00080000:3.0:1787763952.701235:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:2.0:1787763952.701238:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763952.701241:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763952.701246:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:3.0:1787763952.701247:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:2.0:1787763952.701249:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
02000000:00000001:3.0:1787763952.701252:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:2.0:1787763952.701254:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
02000000:00000001:3.0:1787763952.701259:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763952.701260:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832676352, portal 10
00000100:00000001:3.0:1787763952.701264:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787763952.701265:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763952.701268:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049854400 : -115096659697216 : ffff9751f87be1c0)
00000008:00100000:3.0:1787763952.701270:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000040:2.0:1787763952.701276:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87be1c0 x1874599832676352/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:3.0:1787763952.701277:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763952.701282:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763952.701289:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000010:3.0:1787763952.701290:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bed00.
00000100:00000040:2.0:1787763952.701296:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000020:00000040:3.0:1787763952.701296:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000400:00000010:2.0:1787763952.701301:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
02000000:00000001:3.0:1787763952.701302:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763952.701305:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763952.701308:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832676352, offset 0
00000100:00000001:3.0:1787763952.701309:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763952.701313:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763952.701316:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763952.701318:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
02000000:00000010:3.0:1787763952.701324:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613000.
00000400:00000200:2.0:1787763952.701325:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763952.701332:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763952.701335:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00100000:3.0:1787763952.701341:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bed00 x1874599832676608/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787763952.701342:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763952.701347:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763952.701352:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763952.701358:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763952.701364:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58519:58519:256:4294967295] 192.168.203.110@tcp LPNI seq info [58519:58519:8:4294967295]
00000100:00000001:3.0:1787763952.701366:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1787763952.701370:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763952.701372:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763952.701380:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:3.0:1787763952.701384:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000079751c9a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bed00 x1874599832676608/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1787763952.701389:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763952.701395:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000800:00000200:2.0:1787763952.701403:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787763952.701405:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787763952.701406:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787763952.701413:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:3.0:1787763952.701414:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21677)
00000800:00000200:2.1:1787763952.701415:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1787763952.701422:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763952.701424:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000400:00000200:2.0:1787763952.701426:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763952.701430:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763952.701432:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:2.0:1787763952.701434:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:2.0:1787763952.701437:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763952.701440:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87be1c0 x1874599832676352/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701446:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763952.701448:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87be1c0 x1874599832676352/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763977 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701452:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763952.701453:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763952.701456:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763952.701458:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787763952.701461:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763952.701462:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316339744
00000100:00000001:2.0:1787763952.701464:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316340979
00000100:00000001:2.0:1787763952.701466:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316339744
00000100:00000001:2.0:1787763952.701467:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316340979
00000100:00000001:2.0:1787763952.701468:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763952.701470:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763952.701472:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763952.701474:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763952/real 1787763952]  req@ffff9751f87be1c0 x1874599832676352/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763952.701479:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763952.701480:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763952.701481:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763952.701483:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763952.701484:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:2.0:1787763952.701485:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763952.701487:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87be1c0 x1874599832676352/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763952.701493:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87be1c0 x1874599832676352/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701496:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763952.701497:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.701498:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763952.701499:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763952.701501:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87be1c0 x1874599832676352/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701505:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763952.701507:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87be1c0 x1874599832676352/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701512:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763952.701513:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.701514:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763952.701516:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.701517:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763952.701526:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763952.701528:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763952.701531:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763952.701532:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763952.701535:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87be1c0 x1874599832676352/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763952.701540:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832676352:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763952.701542:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763952.701544:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc000 x1874599832676480/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701548:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763952.701549:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763952.701550:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763952.701551:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763952.701552:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763952.701573:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832676480:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763952.701576:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763952.701577:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763952.701578:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763952.701579:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763952.701582:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763952.701584:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763952.701586:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763952.701588:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000400:00000010:2.0:1787763952.701590:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000200:2.0:1787763952.701592:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832676480, portal 10
00000100:00000001:2.0:1787763952.701593:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763952.701594:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049845760 : -115096659705856 : ffff9751f87bc000)
00000100:00000040:2.0:1787763952.701599:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc000 x1874599832676480/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701604:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763952.701606:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763952.701607:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000200:2.0:1787763952.701608:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832676480, offset 0
00000400:00000200:2.0:1787763952.701611:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763952.701614:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763952.701616:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763952.701618:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763952.701620:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763952.701622:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763952.701624:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763952.701626:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58520:58520:256:4294967295] 192.168.203.110@tcp LPNI seq info [58520:58520:8:4294967295]
00000400:00000200:2.0:1787763952.701628:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763952.701633:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763952.701635:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763952.701637:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000800:00000200:2.0:1787763952.701640:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763952.701642:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787763952.701645:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763952.701647:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1787763952.701649:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763952.701650:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000400:00000200:2.0:1787763952.701653:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763952.701655:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763952.701657:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:2.0:1787763952.701658:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:2.0:1787763952.701660:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763952.701662:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc000 x1874599832676480/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701667:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763952.701669:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc000 x1874599832676480/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763977 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701673:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763952.701674:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763952.701677:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763952.701679:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787763952.701681:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763952.701682:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316339732
00000100:00000001:2.0:1787763952.701684:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316340979
00000100:00000001:2.0:1787763952.701685:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316339732
00000100:00000001:2.0:1787763952.701687:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316340979
00000100:00000001:2.0:1787763952.701688:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763952.701689:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763952.701690:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763952.701692:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763952/real 1787763952]  req@ffff9751f87bc000 x1874599832676480/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763952.701697:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763952.701698:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763952.701699:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763952.701701:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763952.701702:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:2.0:1787763952.701703:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763952.701705:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc000 x1874599832676480/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763952.701710:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc000 x1874599832676480/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701713:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763952.701714:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.701715:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763952.701716:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763952.701718:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc000 x1874599832676480/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701723:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763952.701725:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc000 x1874599832676480/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701730:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763952.701732:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.701733:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763952.701734:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.701735:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763952.701736:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763952.701737:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763952.701739:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763952.701740:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763952.701743:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc000 x1874599832676480/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763952.701747:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832676480:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763952.701749:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.701751:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763952.701752:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87be1c0 x1874599832676352/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701756:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763952.701757:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763952.701759:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763952.701760:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763952.701761:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763952.701762:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763952.701763:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763952.701764:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612c00.
02000000:00000001:2.0:1787763952.701766:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763952.701767:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763952.701768:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87be1c0.
00000100:00000001:2.0:1787763952.701770:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763952.701771:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.701772:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763952.701774:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc000 x1874599832676480/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701779:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763952.701780:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763952.701780:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763952.701782:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763952.701783:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763952.701783:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763952.701784:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763952.701785:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612800.
02000000:00000001:2.0:1787763952.701786:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763952.701787:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763952.701788:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc000.
00000100:00000001:2.0:1787763952.701789:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763952.701790:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.701791:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.701793:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763952.701794:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763952.701795:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763952.701796:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763952.701797:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763952.701799:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bed00 x1874599832676608/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701803:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763952.701804:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763952.701805:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763952.701806:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763952.701808:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763952.701809:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832676608:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763952.701812:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763952.701813:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763952.701814:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763952.701815:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763952.701817:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763952.701819:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763952.701821:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763952.701822:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000400:00000010:2.0:1787763952.701824:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
00000100:00000200:2.0:1787763952.701826:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832676608, portal 4
00000100:00000001:2.0:1787763952.701828:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763952.701829:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049857280 : -115096659694336 : ffff9751f87bed00)
00000100:00000040:2.0:1787763952.701832:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bed00 x1874599832676608/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701836:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763952.701838:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763952.701839:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000200:2.0:1787763952.701841:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832676608, offset 0
00000400:00000200:2.0:1787763952.701843:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763952.701845:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763952.701847:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763952.701849:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763952.701851:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763952.701853:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763952.701854:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763952.701856:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58521:58521:256:4294967295] 192.168.203.110@tcp LPNI seq info [58521:58521:8:4294967295]
00000400:00000200:2.0:1787763952.701859:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763952.701863:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763952.701865:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763952.701867:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000800:00000200:2.0:1787763952.701870:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763952.701872:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787763952.701874:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763952.701876:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1787763952.701878:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763952.701880:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000400:00000200:2.0:1787763952.701881:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763952.701883:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763952.701886:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:2.0:1787763952.701887:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:2.0:1787763952.701888:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763952.701891:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bed00 x1874599832676608/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701895:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763952.701897:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bed00 x1874599832676608/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763977 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701901:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763952.701902:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763952.701904:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763952.701906:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787763952.701907:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763952.701908:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316339749
00000100:00000001:2.0:1787763952.701910:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316340979
00000100:00000001:2.0:1787763952.701912:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316339749
00000100:00000001:2.0:1787763952.701913:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316340979
00000100:00000001:2.0:1787763952.701914:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763952.701916:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763952.701917:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763952.701919:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763952/real 1787763952]  req@ffff9751f87bed00 x1874599832676608/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763952.701923:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763952.701924:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763952.701925:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763952.701926:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000400:00000010:2.0:1787763952.701927:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:2.0:1787763952.701928:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763952.701930:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bed00 x1874599832676608/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763952.701939:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bed00 x1874599832676608/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701943:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763952.701944:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.701945:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763952.701946:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763952.701948:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bed00 x1874599832676608/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701952:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763952.701954:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bed00 x1874599832676608/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701958:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763952.701960:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.701961:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763952.701961:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.701962:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763952.701963:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763952.701965:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763952.701967:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763952.701968:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763952.701970:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bed00 x1874599832676608/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763952.701975:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832676608:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763952.701977:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.701978:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763952.701980:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bed00 x1874599832676608/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763977 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763952.701990:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763952.701991:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763952.701993:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763952.701996:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763952.701997:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763952.701998:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787763952.702000:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763952.702002:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613000.
02000000:00000001:2.0:1787763952.702004:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763952.702005:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763952.702007:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bed00.
00000100:00000001:2.0:1787763952.702009:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763952.702011:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.702013:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763952.702015:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763952.702017:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763952.702019:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763952.702020:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763953.340212:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763953.340220:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763953.340223:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763953.340226:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763953.340231:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763953.340249:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763953.340252:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763953.340255:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763953.340261:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763953.340263:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763953.340268:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763953.340271:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763953.340287:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763953.340294:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763953.340297:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763953.340302:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763953.340306:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763953.340312:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763953.340316:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763953.340320:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763953.340325:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763953.340346:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763953.340352:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763953.340355:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763953.724241:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763953.724248:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763953.724250:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763953.724255:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763953.724259:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763953.724262:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763953.724265:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763953.724267:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763953.724270:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763953.980242:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316340979, at 4316340992
00000100:00000010:3.0:1787763953.980264:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7540.
00000100:00000040:2.0:1787763953.980293:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316340979, at 4316340992
00000100:00000010:2.0:1787763953.980301:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00000100:00000001:1.0:1787763954.364255:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763954.364262:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763954.364263:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763954.364264:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763954.364270:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763954.364271:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763954.364276:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763954.364278:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763954.364280:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763954.364281:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763954.364284:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763954.364286:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763954.364288:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763954.364289:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763954.364293:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763954.364351:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763954.364355:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763954.364359:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763954.748220:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763954.748228:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763954.748230:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763954.748235:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763954.748238:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763954.748241:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763954.748244:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763954.748247:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763954.748249:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763955.388244:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763955.388247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763955.388250:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763955.388251:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763955.388254:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763955.388257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787763955.388258:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763955.388261:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763955.388262:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763955.388266:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763955.388267:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763955.388270:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763955.388271:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763955.388274:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763955.388276:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763955.388330:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763955.388333:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763955.388336:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763955.772262:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763955.772270:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763955.772272:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763955.772287:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763955.772292:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763955.772295:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763955.772298:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763955.772301:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763955.772304:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763956.412246:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763956.412246:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763956.412252:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763956.412253:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763956.412257:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763956.412258:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763956.412263:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00100000:1.0:1787763956.412264:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763956.412265:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:1.0:1787763956.412270:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763956.412271:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763956.412275:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:1.0:1787763956.412276:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763956.412278:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763956.412281:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763956.412282:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763956.412285:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763956.412286:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763956.412289:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763956.412291:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763956.412292:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763956.412352:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763956.412356:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763956.412359:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763956.796237:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763956.796243:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763956.796245:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763956.796249:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763956.796253:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763956.796256:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763956.796259:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763956.796261:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763956.796264:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763957.436243:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763957.436246:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763957.436251:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763957.436251:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763957.436254:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763957.436257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787763957.436259:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763957.436260:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763957.436264:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763957.436266:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763957.436269:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763957.436269:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763957.436272:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763957.436275:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763957.436277:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763957.436324:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763957.436329:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763957.436331:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763957.820210:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1787763957.820214:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763957.820217:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763957.820221:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763957.820222:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00100000:2.0:1787763957.820226:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763957.820226:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763957.820231:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763957.820232:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763957.820236:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763957.820237:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763957.820239:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763957.820242:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1787763957.820244:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21672
00000100:00000001:2.0:1787763957.820245:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763957.820254:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763957.820259:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763957.820262:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763957.820265:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763957.820268:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763957.820269:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763957.820271:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763957.820274:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763957.820275:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787763957.820277:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763957.820280:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787763957.820282:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787763957.820283:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787763957.820285:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763957.820287:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763957.820288:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763957.820289:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763957.820291:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787763957.820293:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763957.820295:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763957.820297:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763957.820300:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763957.820302:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763957.820303:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787763957.820305:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763957.820313:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bf480.
00000020:00000040:3.0:1787763957.820315:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787763957.820319:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763957.820320:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763957.820322:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763957.820324:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763957.820327:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763957.820333:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613400.
00000100:00000001:3.0:1787763957.820338:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763957.820346:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bf480 x1874599832676736/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787763957.820354:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787763957.820357:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763957.820361:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c2965388] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bf480 x1874599832676736/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763957.820377:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763957.820379:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787763957.820382:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787763957.820383:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763957.820385:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763957.820387:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787763957.820390:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21672
00000400:00000200:3.0:1787763957.820392:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763957.820394:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763957.820396:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787763957.820397:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1787763957.820400:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763957.820402:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763957.820403:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763957.820405:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763957.820407:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763957.820410:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763957.820413:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000040:2.0:1787763957.820414:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bf480 x1874599832676736/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787763957.820415:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:3.0:1787763957.820418:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763957.820421:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:3.0:1787763957.820426:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:3.0:1787763957.820430:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787763957.820432:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763957.820436:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763957.820436:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763957.820442:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1787763957.820443:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:3.0:1787763957.820445:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:2.0:1787763957.820448:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763957.820448:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763957.820454:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:2.0:1787763957.820455:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763957.820459:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00100000:2.0:1787763957.820463:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832676736:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787763957.820464:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763957.820468:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787763957.820471:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763957.820474:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763957.820475:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763957.820479:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763957.820480:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763957.820482:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1787763957.820484:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:3.0:1787763957.820487:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000010:2.0:1787763957.820491:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
00000008:00000001:3.0:1787763957.820491:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763957.820492:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763957.820496:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763957.820499:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bed00.
00000400:00000200:2.0:1787763957.820503:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:3.0:1787763957.820504:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787763957.820509:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763957.820510:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763957.820511:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000100:00000001:3.0:1787763957.820512:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763957.820515:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763957.820517:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763957.820529:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
02000000:00000010:3.0:1787763957.820535:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610000.
00000100:00000200:2.0:1787763957.820538:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832676736, portal 25
00000100:00000001:3.0:1787763957.820540:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763957.820542:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763957.820545:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049859200 : -115096659692416 : ffff9751f87bf480)
00000100:00100000:3.0:1787763957.820548:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bed00 x1874599832676864/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:2.0:1787763957.820554:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bf480 x1874599832676736/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763957.820560:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787763957.820564:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763957.820570:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763957.820571:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000079751c9a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bed00 x1874599832676864/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787763957.820578:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763957.820584:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec550.
00000100:00000200:2.0:1787763957.820589:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832676736, offset 0
00000100:00000001:3.0:1787763957.820598:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763957.820599:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00080000:3.0:1787763957.820600:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787763957.820604:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1787763957.820605:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763957.820607:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1787763957.820610:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763957.820611:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787763957.820616:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21672
00000400:00000200:2.0:1787763957.820621:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763957.820625:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763957.820629:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763957.820631:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763957.820634:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763957.820637:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58522:58522:256:4294967295] 192.168.203.110@tcp LPNI seq info [58522:58522:8:4294967295]
00000400:00000200:2.0:1787763957.820641:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763957.820647:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000400:00000200:3.0:1787763957.820654:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:2.0:1787763957.820655:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763957.820660:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000800:00000010:2.0:1787763957.820661:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f500.
00000100:00080000:3.0:1787763957.820664:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763957.820667:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763957.820672:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1787763957.820673:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:3.0:1787763957.820676:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763957.820679:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763957.820683:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:2.1:1787763957.820683:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:3.0:1787763957.820685:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:3.0:1787763957.820689:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763957.820691:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:3.0:1787763957.820693:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:3.0:1787763957.820694:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:3.0:1787763957.820697:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763957.820698:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763957.820699:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763957.820700:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763957.820701:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:3.0:1787763957.820704:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763957.820706:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763957.820706:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763957.820712:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1787763957.820715:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:3.0:1787763957.820716:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763957.820718:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316346099
02000000:00000001:3.0:1787763957.820719:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763957.820724:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000010:2.0:1787763957.820727:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000008:00100000:3.0:1787763957.820727:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787763957.820730:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763957.820733:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763957.820735:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000010:3.0:1787763957.820738:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc000.
00000020:00000040:3.0:1787763957.820741:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000001:2.0:1787763957.820744:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316346099
02000000:00000001:3.0:1787763957.820744:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763957.820747:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763957.820750:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000010:2.0:1787763957.820751:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7780.
02000000:00000001:3.0:1787763957.820752:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763957.820755:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763957.820757:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
02000000:00000010:3.0:1787763957.820760:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611000.
00000100:00000001:2.0:1787763957.820761:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763957.820766:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763957.820767:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763957.820770:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763957.820774:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc000 x1874599832676992/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763957.820775:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763957.820784:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763957.820784:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763957.820788:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1787763957.820789:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763957.820794:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763957.820797:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1787763957.820797:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3fc1c6c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc000 x1874599832676992/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763957.820800:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763957.820807:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bed00 x1874599832676864/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763957.820809:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763957.820814:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763957.820819:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:3.0:1787763957.820820:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1787763957.820821:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763957.820824:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:2.0:1787763957.820825:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763957.820829:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:3.0:1787763957.820830:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
02000000:00000001:2.0:1787763957.820834:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763957.820834:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763957.820838:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1787763957.820841:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832676864:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763957.820842:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763957.820846:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:3.0:1787763957.820848:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21672
02000000:00000001:2.0:1787763957.820850:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763957.820853:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763957.820853:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:2.0:1787763957.820857:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000400:00000200:3.0:1787763957.820859:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763957.820862:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
02000000:00000010:2.0:1787763957.820863:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
00000100:00000040:3.0:1787763957.820864:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:2.0:1787763957.820868:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763957.820868:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763957.820873:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787763957.820877:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763957.820878:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763957.820881:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000010:2.0:1787763957.820882:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000020:00000040:3.0:1787763957.820883:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:3.0:1787763957.820887:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:2.0:1787763957.820888:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000020:00000040:3.0:1787763957.820889:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:3.0:1787763957.820894:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000200:2.0:1787763957.820895:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832676864, portal 10
00000020:00000001:3.0:1787763957.820896:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:2.0:1787763957.820899:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000040:3.0:1787763957.820902:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763957.820903:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049857280 : -115096659694336 : ffff9751f87bed00)
00000100:00000001:3.0:1787763957.820905:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763957.820908:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763957.820910:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763957.820913:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000040:2.0:1787763957.820914:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bed00 x1874599832676864/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787763957.820917:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763957.820922:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763957.820925:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787763957.820929:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:3.0:1787763957.820929:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763957.820932:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763957.820936:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787763957.820936:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787763957.820939:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000400:00000010:2.0:1787763957.820940:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ece58.
00000008:00000001:3.0:1787763957.820941:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763957.820945:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832676864, offset 0
00000100:00000001:3.0:1787763957.820946:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763957.820952:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87be1c0.
00000400:00000200:2.0:1787763957.820956:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000020:00000040:3.0:1787763957.820956:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:3.0:1787763957.820960:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:2.0:1787763957.820962:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:3.0:1787763957.820963:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763957.820967:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763957.820969:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1787763957.820970:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
02000000:00000001:3.0:1787763957.820972:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763957.820974:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763957.820978:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
02000000:00000010:3.0:1787763957.820978:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611400.
00000400:00000200:2.0:1787763957.820981:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1787763957.820983:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763957.820986:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00100000:3.0:1787763957.820991:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87be1c0 x1874599832677120/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787763957.820992:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58523:58523:255:4294967295] 192.168.203.110@tcp LPNI seq info [58523:58523:7:4294967295]
00000400:00000200:2.0:1787763957.821004:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763957.821009:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787763957.821013:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763957.821018:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:3.0:1787763957.821020:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e2011c6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87be1c0 x1874599832677120/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1787763957.821028:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763957.821035:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f700.
00000100:00000001:3.0:1787763957.821035:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763957.821043:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21682)
00000800:00000200:2.0:1787763957.821045:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763957.821050:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787763957.821051:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763957.821054:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787763957.821058:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763957.821061:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316344863
00000100:00000001:2.0:1787763957.821064:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316346099
00000100:00000001:2.0:1787763957.821067:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316344863
00000100:00000001:2.0:1787763957.821069:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316346099
00000100:00000001:2.0:1787763957.821071:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763957.821073:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763957.821076:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763957.821078:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763957.821081:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763957.821083:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1787763957.821085:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763957.821086:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763957.821087:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763957.821091:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc000 x1874599832676992/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763957.821099:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763957.821100:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763957.821102:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763957.821104:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763957.821106:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763957.821109:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832676992:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763957.821112:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763957.821114:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763957.821116:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763957.821118:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763957.821121:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763957.821123:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763957.821126:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763957.821129:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da04d0.
00000400:00000010:2.0:1787763957.821132:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787763957.821153:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832676992, portal 10
00000100:00000001:2.0:1787763957.821156:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763957.821157:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049845760 : -115096659705856 : ffff9751f87bc000)
00000100:00000040:2.0:1787763957.821163:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc000 x1874599832676992/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763957.821170:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763957.821172:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763957.821175:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec7f8.
00000100:00000200:2.0:1787763957.821178:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832676992, offset 0
00000400:00000200:2.0:1787763957.821182:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763957.821186:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763957.821190:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763957.821193:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763957.821196:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763957.821198:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763957.821200:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763957.821203:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58524:58524:254:4294967295] 192.168.203.110@tcp LPNI seq info [58524:58524:6:4294967295]
00000400:00000200:2.0:1787763957.821207:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763957.821214:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763957.821219:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763957.821223:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f900.
00000800:00000200:2.0:1787763957.821227:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763957.821231:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787763957.821234:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763957.821237:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787763957.821241:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763957.821243:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316344852
00000100:00000001:2.0:1787763957.821246:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316346099
00000100:00000001:2.0:1787763957.821248:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316344852
00000100:00000001:2.0:1787763957.821250:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316346099
00000100:00000001:2.0:1787763957.821252:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763957.821255:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763957.821257:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763957.821261:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87be1c0 x1874599832677120/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763957.821274:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763957.821275:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763957.821277:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763957.821279:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763957.821281:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763957.821284:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832677120:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763957.821287:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763957.821289:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763957.821291:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763957.821292:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763957.821296:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763957.821298:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763957.821301:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763957.821303:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787763957.821307:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763957.821310:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832677120, portal 4
00000100:00000001:2.0:1787763957.821311:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763957.821313:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049854400 : -115096659697216 : ffff9751f87be1c0)
00000100:00000040:2.0:1787763957.821318:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87be1c0 x1874599832677120/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763957.821324:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763957.821326:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763957.821329:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec660.
00000100:00000200:2.0:1787763957.821331:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832677120, offset 0
00000400:00000200:2.0:1787763957.821336:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763957.821339:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763957.821343:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763957.821346:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763957.821348:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763957.821350:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763957.821353:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763957.821356:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58525:58525:253:4294967295] 192.168.203.110@tcp LPNI seq info [58525:58525:5:4294967295]
00000400:00000200:2.0:1787763957.821360:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763957.821366:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763957.821370:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763957.821374:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f300.
00000800:00000200:2.0:1787763957.821378:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763957.821382:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787763957.821385:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763957.821387:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787763957.821390:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763957.821392:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316344869
00000100:00000001:2.0:1787763957.821394:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316346099
00000100:00000001:2.0:1787763957.821396:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316344869
00000100:00000001:2.0:1787763957.821398:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316346099
00000100:00000001:2.0:1787763957.821400:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763957.821402:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763957.821405:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763957.821407:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763957.821411:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763957.821412:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1787763957.821414:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763957.821415:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763957.821417:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763957.821419:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763958.460229:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763958.460235:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763958.460237:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763958.460239:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787763958.460244:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763958.460244:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763958.460250:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763958.460250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763958.460254:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763958.460255:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763958.460257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763958.460258:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763958.460260:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763958.460262:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763958.460266:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763958.460302:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763958.460305:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763958.460308:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763958.844217:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763958.844223:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763958.844224:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763958.844229:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763958.844232:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763958.844234:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763958.844236:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763958.844238:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763958.844241:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763959.100228:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316346099, at 4316346112
00000100:00000010:3.0:1787763959.100242:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7780.
00000100:00000040:2.0:1787763959.100248:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316346099, at 4316346112
00000100:00000010:2.0:1787763959.100257:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00010000:00000001:3.0:1787763959.484205:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763959.484211:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763959.484216:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763959.484218:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763959.484221:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763959.484222:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763959.484252:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763959.484259:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763959.484262:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763959.484267:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000100:00100000:1.0:1787763959.484267:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000020:00000001:0.0:1787763959.484271:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763959.484274:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00010000:0.0:1787763959.484276:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787763959.484280:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763959.484285:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763959.484288:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763959.484291:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763959.868270:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763959.868278:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763959.868280:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763959.868286:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763959.868290:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763959.868295:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763959.868298:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763959.868301:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763959.868304:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763960.508223:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763960.508228:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763960.508229:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763960.508233:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763960.508236:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763960.508237:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763960.508240:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763960.508242:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763960.508244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763960.508246:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763960.508247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763960.508249:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763960.508252:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763960.508254:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763960.508257:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763960.508325:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763960.508328:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763960.508331:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763960.892271:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763960.892277:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763960.892279:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763960.892283:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763960.892287:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763960.892290:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763960.892292:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763960.892295:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763960.892297:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:3.0:1787763960.892421:0:257565:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.110:988
00000400:00000100:3.0:1787763960.892458:0:257565:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.110@tcp at host 192.168.203.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:3.0:1787763960.892466:0:257565:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763960.892469:0:257565:0:(router.c:1791:lnet_notify()) 192.168.203.10@tcp notifying 192.168.203.110@tcp: down
00000400:00000200:3.0:1787763960.892472:0:257565:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763960.892476:0:257565:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:3.0:1787763960.892479:0:257565:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:3.0:1787763960.892482:0:257565:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f500.
00000400:00000200:3.0:1787763960.892486:0:257565:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:3.0:1787763960.892490:0:257565:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:3.0:1787763960.892494:0:257565:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec550
00000400:00000010:3.0:1787763960.892496:0:257565:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec550.
00000100:00000001:3.0:1787763960.892514:0:257565:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763960.892522:0:257565:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f87bf480 x1874599832676736/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763960.892543:0:257565:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763960.892549:0:257565:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bf480 x1874599832676736/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763982 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763960.892557:0:257565:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763960.892559:0:257565:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:3.0:1787763960.892570:0:257565:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:3.0:1787763960.892573:0:257565:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f700.
00000400:00000200:3.0:1787763960.892577:0:257565:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00000001:2.0:1787763960.892578:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:3.0:1787763960.892583:0:257565:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000100:00000001:2.0:1787763960.892584:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000400:00000200:3.0:1787763960.892588:0:257565:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ece58
00000100:00000001:2.0:1787763960.892590:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000400:00000010:3.0:1787763960.892593:0:257565:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ece58.
00000100:00000001:3.0:1787763960.892597:0:257565:0:(events.c:34:request_out_callback()) Process entered
00000100:00100000:2.0:1787763960.892602:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763957/real 1787763960]  req@ffff9751f87bf480 x1874599832676736/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763960.892606:0:257565:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f87bed00 x1874599832676864/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763960.892630:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:3.0:1787763960.892632:0:257565:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000400:00000001:2.0:1787763960.892635:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:3.0:1787763960.892638:0:257565:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bed00 x1874599832676864/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763982 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1787763960.892641:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787763960.892648:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000100:00000001:3.0:1787763960.892650:0:257565:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763960.892652:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:3.0:1787763960.892654:0:257565:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763960.892660:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000800:00000100:3.0:1787763960.892665:0:257565:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000100:00000200:2.0:1787763960.892670:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bf480 x1874599832676736/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:3.0:1787763960.892671:0:257565:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f900.
00000100:00000200:2.0:1787763960.892687:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bf480 x1874599832676736/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787763960.892692:0:257565:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00000001:2.0:1787763960.892699:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000400:00000200:3.0:1787763960.892703:0:257565:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000100:00000001:2.0:1787763960.892704:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:3.0:1787763960.892709:0:257565:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec7f8
00000100:00000001:2.0:1787763960.892710:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763960.892714:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000400:00000010:3.0:1787763960.892714:0:257565:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec7f8.
00000100:00100000:2.0:1787763960.892719:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bf480 x1874599832676736/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763960.892721:0:257565:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:2.0:1787763960.892728:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000200:3.0:1787763960.892731:0:257565:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f87bc000 x1874599832676992/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787763960.892739:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bf480 x1874599832676736/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763960.892747:0:257565:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787763960.892755:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:3.0:1787763960.892755:0:257565:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc000 x1874599832676992/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763982 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763960.892763:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763960.892767:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763960.892771:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763960.892772:0:257565:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763960.892776:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763960.892776:0:257565:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763960.892781:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000800:00000100:3.0:1787763960.892786:0:257565:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000100:00080000:2.0:1787763960.892788:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000800:00000010:3.0:1787763960.892790:0:257565:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f300.
00000400:00000200:3.0:1787763960.892796:0:257565:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00080000:2.0:1787763960.892798:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000400:00000200:3.0:1787763960.892805:0:257565:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000100:00000001:2.0:1787763960.892806:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000400:00000200:3.0:1787763960.892810:0:257565:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec660
00000400:00000010:3.0:1787763960.892814:0:257565:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec660.
00000100:00000040:2.0:1787763960.892816:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bf480 x1874599832676736/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763960.892818:0:257565:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763960.892827:0:257565:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f87be1c0 x1874599832677120/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763960.892833:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832676736:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787763960.892842:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763960.892848:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763960.892854:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bf480 x1874599832676736/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763960.892854:0:257565:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763960.892862:0:257565:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87be1c0 x1874599832677120/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763982 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763960.892868:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763960.892873:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000001:3.0:1787763960.892875:0:257565:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763960.892878:0:257565:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000010:2.0:1787763960.892879:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763960.892887:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763960.892889:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763960.892891:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787763960.892894:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763960.892896:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613400.
02000000:00000001:2.0:1787763960.892900:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763960.892902:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763960.892904:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bf480.
00000100:00000001:2.0:1787763960.892908:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763960.892909:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763960.892912:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763960.892914:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763960.892915:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763960.892916:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763960.892920:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763957/real 1787763960]  req@ffff9751f87bc000 x1874599832676992/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763960.892929:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763960.892930:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763960.892932:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da04d0.
00000400:00000200:2.0:1787763960.892935:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787763960.892936:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787763960.892939:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763960.892942:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc000 x1874599832676992/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763960.892950:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc000 x1874599832676992/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763960.892955:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763960.892957:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763960.892958:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763960.892959:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763960.892963:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc000 x1874599832676992/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763960.892969:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763960.892972:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc000 x1874599832676992/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763960.892977:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763960.892980:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763960.892981:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763960.892982:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763960.892984:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763960.892985:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763960.892988:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763960.892992:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763960.892994:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763960.892997:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc000 x1874599832676992/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763960.893004:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832676992:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763960.893008:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763960.893011:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763957/real 1787763960]  req@ffff9751f87be1c0 x1874599832677120/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763960.893018:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763960.893019:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763960.893021:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787763960.893023:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763960.893024:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763960.893027:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763960.893029:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87be1c0 x1874599832677120/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763960.893037:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87be1c0 x1874599832677120/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763960.893042:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763960.893044:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763960.893045:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763960.893047:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763960.893050:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87be1c0 x1874599832677120/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763960.893056:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763960.893060:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87be1c0 x1874599832677120/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763960.893078:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763960.893080:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763960.893082:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763960.893084:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763960.893085:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763960.893087:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763960.893089:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763960.893093:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763960.893096:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763960.893101:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87be1c0 x1874599832677120/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763960.893108:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832677120:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763960.893112:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763960.893116:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763957/real 1787763960]  req@ffff9751f87bed00 x1874599832676864/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763960.893123:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763960.893125:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763960.893126:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763960.893129:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763960.893131:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763960.893155:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763960.893160:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bed00 x1874599832676864/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763960.893169:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bed00 x1874599832676864/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763960.893176:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763960.893179:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763960.893181:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763960.893182:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763960.893187:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bed00 x1874599832676864/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763960.893194:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763960.893199:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bed00 x1874599832676864/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763960.893207:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763960.893210:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763960.893212:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763960.893213:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763960.893215:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763960.893234:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763960.893237:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763960.893242:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763960.893244:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763960.893249:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bed00 x1874599832676864/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763960.893258:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832676864:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763960.893262:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763960.893264:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763960.893268:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc000 x1874599832676992/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763960.893276:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763960.893277:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763960.893280:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763960.893283:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763960.893285:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763960.893287:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763960.893290:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763960.893291:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611000.
02000000:00000001:2.0:1787763960.893294:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763960.893296:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763960.893298:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc000.
00000100:00000001:2.0:1787763960.893301:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763960.893302:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763960.893304:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763960.893306:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87be1c0 x1874599832677120/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763960.893311:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763960.893312:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763960.893313:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763960.893314:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763960.893315:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763960.893316:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787763960.893318:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763960.893319:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611400.
02000000:00000001:2.0:1787763960.893321:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763960.893322:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763960.893324:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87be1c0.
00000100:00000001:2.0:1787763960.893326:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763960.893327:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763960.893329:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763960.893333:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bed00 x1874599832676864/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763982 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763960.893339:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763960.893340:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763960.893342:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763960.893344:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763960.893346:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763960.893347:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763960.893349:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763960.893351:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610000.
02000000:00000001:2.0:1787763960.893354:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763960.893355:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763960.893357:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bed00.
00000100:00000001:2.0:1787763960.893359:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763960.893361:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763960.893362:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763960.893370:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763960.893372:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763960.893374:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763960.893376:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763961.532237:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763961.532242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763961.532244:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763961.532248:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763961.532249:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763961.532256:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763961.532258:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763961.532260:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763961.532261:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763961.532266:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763961.532269:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:1.0:1787763961.532270:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763961.532273:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:1.0:1787763961.532274:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763961.532277:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763961.532279:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763961.532280:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763961.532284:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763961.532287:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763961.532290:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763961.532293:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763961.532352:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763961.532356:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763961.532358:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787763961.916236:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787763961.916243:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763961.916246:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763961.916251:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763961.916255:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763961.916259:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763961.916261:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763961.916270:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763961.916273:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763962.556240:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763962.556250:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763962.556253:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763962.556259:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763962.556263:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763962.556268:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763962.556269:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763962.556275:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763962.556276:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763962.556279:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763962.556286:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763962.556288:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763962.556294:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763962.556300:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763962.556301:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763962.556428:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763962.556434:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763962.556438:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787763962.940342:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787763962.940355:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787763962.940362:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787763962.940364:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1787763962.940368:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763962.940370:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763962.940375:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763962.940379:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00100000:2.0:1787763962.940382:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763962.940387:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763962.940391:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1787763962.940393:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21677
00000100:00000001:2.0:1787763962.940395:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763962.940398:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.940402:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763962.940407:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763962.940418:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763962.940423:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763962.940427:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763962.940431:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763962.940434:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763962.940436:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763962.940441:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763962.940444:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787763962.940448:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763962.940452:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787763962.940455:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787763962.940458:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787763962.940461:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763962.940464:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763962.940466:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763962.940468:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763962.940472:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787763962.940476:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763962.940479:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763962.940496:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763962.940501:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763962.940504:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763962.940507:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787763962.940510:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763962.940521:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bde00.
00000020:00000040:3.0:1787763962.940525:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787763962.940530:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763962.940532:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763962.940535:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763962.940539:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763962.940545:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763962.940555:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613000.
00000100:00000001:3.0:1787763962.940564:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763962.940577:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bde00 x1874599832677248/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787763962.940595:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787763962.940605:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763962.940611:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000941163c2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bde00 x1874599832677248/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763962.940642:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763962.940646:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787763962.940650:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787763962.940652:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763962.940653:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763962.940656:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787763962.940660:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21677
00000400:00000200:3.0:1787763962.940664:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763962.940667:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763962.940669:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763962.940672:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763962.940675:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763962.940677:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763962.940679:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763962.940682:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763962.940683:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:3.0:1787763962.940685:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763962.940687:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:3.0:1787763962.940690:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:3.0:1787763962.940693:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:3.0:1787763962.940696:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763962.940698:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763962.940701:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763962.940703:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763962.940706:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:3.0:1787763962.940709:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763962.940712:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763962.940714:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763962.940722:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763962.940726:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763962.940729:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787763962.940730:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000008:00100000:3.0:1787763962.940734:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1787763962.940738:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000008:00000001:3.0:1787763962.940741:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763962.940743:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1787763962.940745:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763962.940751:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bfc00.
00000100:00000040:2.0:1787763962.940755:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bde00 x1874599832677248/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787763962.940756:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787763962.940762:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763962.940765:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763962.940770:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763962.940772:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763962.940776:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763962.940777:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763962.940782:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763962.940782:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612800.
00000100:00000001:3.0:1787763962.940791:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763962.940793:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763962.940799:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00100000:3.0:1787763962.940800:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bfc00 x1874599832677376/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1787763962.940805:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763962.940815:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832677248:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787763962.940816:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787763962.940821:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763962.940827:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:3.0:1787763962.940830:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003c1f6b31] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bfc00 x1874599832677376/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787763962.940835:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763962.940841:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763962.940847:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763962.940855:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
00000100:00000001:3.0:1787763962.940857:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763962.940862:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763962.940863:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787763962.940869:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1787763962.940870:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:2.0:1787763962.940874:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:3.0:1787763962.940874:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763962.940879:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:2.0:1787763962.940884:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000100:00080000:3.0:1787763962.940885:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21677
00000400:00000200:3.0:1787763962.940892:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763962.940896:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000400:00000200:3.0:1787763962.940896:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763962.940901:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763962.940906:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763962.940909:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763962.940911:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763962.940914:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763962.940916:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763962.940917:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:3.0:1787763962.940919:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763962.940921:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:3.0:1787763962.940923:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:3.0:1787763962.940926:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:3.0:1787763962.940929:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763962.940931:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763962.940933:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763962.940935:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763962.940938:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:3.0:1787763962.940941:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763962.940944:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763962.940947:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763962.940950:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763962.940953:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763962.940955:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787763962.940958:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787763962.940961:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763962.940963:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763962.940968:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc3c0.
00000100:00000200:2.0:1787763962.940969:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832677248, portal 25
00000020:00000040:3.0:1787763962.940973:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000001:2.0:1787763962.940975:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:3.0:1787763962.940978:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1787763962.940979:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049853440 : -115096659698176 : ffff9751f87bde00)
02000000:00000001:3.0:1787763962.940982:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763962.940986:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763962.940988:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763962.940991:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763962.940996:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bde00 x1874599832677248/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1787763962.940997:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612c00.
00000100:00000001:3.0:1787763962.941004:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763962.941014:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc3c0 x1874599832677504/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763962.941015:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763962.941024:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763962.941029:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763962.941033:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832677248, offset 0
00000400:00000200:2.0:1787763962.941042:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763962.941053:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763962.941060:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763962.941068:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763962.941073:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763962.941078:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763962.941081:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763962.941086:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58526:58526:256:4294967295] 192.168.203.110@tcp LPNI seq info [58526:58526:8:4294967295]
00000400:00000200:2.0:1787763962.941091:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763962.941102:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763962.941111:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763962.941118:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000100:00000001:3.0:1787763962.941129:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:2.0:1787763962.941130:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787763962.941171:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787763962.941177:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000100:00000040:3.0:1787763962.941179:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000060bf8470] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc3c0 x1874599832677504/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763962.941191:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763962.941195:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1787763962.941200:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:3.0:1787763962.941204:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787763962.941208:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787763962.941211:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763962.941213:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763962.941216:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787763962.941222:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21677
00000400:00000200:3.0:1787763962.941230:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:2.1:1787763962.941231:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000400:00000200:3.0:1787763962.941237:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763962.941241:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763962.941245:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763962.941246:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763962.941248:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763962.941250:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:2.1:1787763962.941252:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:3.0:1787763962.941253:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763962.941256:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:3.0:1787763962.941260:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763962.941263:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000800:00000100:2.0:1787763962.941264:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000020:00000001:3.0:1787763962.941267:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000800:00000010:2.0:1787763962.941270:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000020:00000001:3.0:1787763962.941272:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000200:2.0:1787763962.941279:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1787763962.941279:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763962.941283:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763962.941287:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787763962.941293:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1787763962.941293:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763962.941299:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:3.0:1787763962.941304:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:2.0:1787763962.941305:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000100:00000001:3.0:1787763962.941307:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763962.941311:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:2.0:1787763962.941312:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
02000000:00000001:3.0:1787763962.941315:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763962.941319:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763962.941320:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:3.0:1787763962.941322:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787763962.941327:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000200:2.0:1787763962.941330:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bde00 x1874599832677248/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:3.0:1787763962.941331:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763962.941336:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763962.941342:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc780.
00000020:00000040:3.0:1787763962.941349:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:2.0:1787763962.941352:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:3.0:1787763962.941353:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763962.941357:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763962.941361:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bde00 x1874599832677248/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763987 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763962.941362:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763962.941365:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763962.941368:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763962.941376:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612400.
00000100:00000001:2.0:1787763962.941379:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763962.941382:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763962.941384:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00100000:3.0:1787763962.941392:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc780 x1874599832677632/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763962.941395:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763962.941405:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763962.941406:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763962.941408:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316351220
00000100:00000001:3.0:1787763962.941412:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763962.941416:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00000040:3.0:1787763962.941421:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7d56184] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc780 x1874599832677632/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.941426:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787763962.941436:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763962.941437:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316351220
00000100:00000010:2.0:1787763962.941443:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7360.
00000100:00000040:3.0:1787763962.941443:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21687)
00000100:00000001:2.0:1787763962.941451:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787763962.941455:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763962.941457:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763962.941459:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763962.941464:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763962/real 1787763962]  req@ffff9751f87bde00 x1874599832677248/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763962.941472:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763962.941474:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763962.941476:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763962.941478:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763962.941479:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763962.941481:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763962.941516:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bde00 x1874599832677248/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763962.941523:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bde00 x1874599832677248/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.941533:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763962.941535:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.941537:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763962.941539:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763962.941543:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bde00 x1874599832677248/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.941551:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763962.941557:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bde00 x1874599832677248/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.941566:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763962.941570:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.941572:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763962.941573:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.941576:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763962.941579:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763962.941582:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763962.941587:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763962.941590:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763962.941594:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bde00 x1874599832677248/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763962.941603:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832677248:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787763962.941635:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.941637:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763962.941641:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bde00 x1874599832677248/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.941646:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763962.941648:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763962.941651:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763962.941654:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763962.941655:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763962.941657:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787763962.941659:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763962.941664:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613000.
02000000:00000001:2.0:1787763962.941669:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763962.941671:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763962.941673:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bde00.
00000100:00000001:2.0:1787763962.941677:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763962.941679:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.941681:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.941690:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763962.941692:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763962.941695:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763962.941697:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763962.941699:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763962.941704:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bfc00 x1874599832677376/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.941713:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763962.941715:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763962.941718:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763962.941721:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763962.941723:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763962.941727:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832677376:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763962.941731:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763962.941734:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763962.941736:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763962.941738:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763962.941741:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763962.941743:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763962.941747:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763962.941749:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763962.941753:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763962.941756:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832677376, portal 10
00000100:00000001:2.0:1787763962.941759:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763962.941760:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049861120 : -115096659690496 : ffff9751f87bfc00)
00000100:00000040:2.0:1787763962.941766:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bfc00 x1874599832677376/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.941771:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763962.941773:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763962.941774:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763962.941776:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832677376, offset 0
00000400:00000200:2.0:1787763962.941779:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763962.941782:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763962.941786:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763962.941788:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763962.941791:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763962.941793:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763962.941795:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763962.941797:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58527:58527:256:4294967295] 192.168.203.110@tcp LPNI seq info [58527:58527:8:4294967295]
00000400:00000200:2.0:1787763962.941800:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763962.941805:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763962.941808:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763962.941810:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763962.941814:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763962.941816:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787763962.941820:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763962.941822:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787763962.941825:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763962.941827:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763962.941829:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763962.941832:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763962.941834:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763962.941835:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763962.941837:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763962.941840:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bfc00 x1874599832677376/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.941870:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763962.941873:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bfc00 x1874599832677376/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763987 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.941878:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763962.941879:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763962.941881:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763962.941884:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787763962.941887:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763962.941888:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316349984
00000100:00000001:2.0:1787763962.941891:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316351220
00000100:00000001:2.0:1787763962.941893:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316349984
00000100:00000001:2.0:1787763962.941894:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316351220
00000100:00000001:2.0:1787763962.941896:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763962.941898:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763962.941899:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763962.941902:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763962/real 1787763962]  req@ffff9751f87bfc00 x1874599832677376/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763962.941908:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763962.941910:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763962.941911:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763962.941913:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763962.941914:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763962.941915:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763962.941917:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bfc00 x1874599832677376/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763962.941923:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bfc00 x1874599832677376/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.941928:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763962.941930:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.941931:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763962.941932:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763962.941935:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bfc00 x1874599832677376/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.941939:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763962.941941:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bfc00 x1874599832677376/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.941946:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763962.941947:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.941948:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763962.941949:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.941950:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763962.941961:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763962.941963:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763962.941965:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763962.941966:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763962.941968:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bfc00 x1874599832677376/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763962.941973:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832677376:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763962.941976:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763962.941978:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc3c0 x1874599832677504/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.941982:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763962.941984:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763962.941985:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763962.941986:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763962.941987:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763962.941990:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832677504:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763962.941993:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763962.941994:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763962.941996:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763962.941997:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763962.942000:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763962.942001:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763962.942003:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763962.942005:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763962.942007:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763962.942009:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832677504, portal 10
00000100:00000001:2.0:1787763962.942011:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763962.942012:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049846720 : -115096659704896 : ffff9751f87bc3c0)
00000100:00000040:2.0:1787763962.942016:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc3c0 x1874599832677504/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.942020:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763962.942022:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763962.942024:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763962.942025:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832677504, offset 0
00000400:00000200:2.0:1787763962.942028:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763962.942030:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763962.942033:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763962.942035:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763962.942037:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763962.942038:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763962.942040:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763962.942043:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58528:58528:256:4294967295] 192.168.203.110@tcp LPNI seq info [58528:58528:8:4294967295]
00000400:00000200:2.0:1787763962.942045:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763962.942050:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763962.942052:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763962.942055:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763962.942058:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763962.942060:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787763962.942063:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763962.942065:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787763962.942067:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763962.942069:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763962.942071:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763962.942074:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763962.942076:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763962.942077:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763962.942079:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763962.942081:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc3c0 x1874599832677504/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.942086:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763962.942088:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc3c0 x1874599832677504/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763987 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.942109:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763962.942111:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763962.942113:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763962.942115:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787763962.942117:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763962.942119:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316349972
00000100:00000001:2.0:1787763962.942121:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316351220
00000100:00000001:2.0:1787763962.942122:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316349972
00000100:00000001:2.0:1787763962.942124:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316351220
00000100:00000001:2.0:1787763962.942126:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763962.942127:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763962.942128:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763962.942131:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763962/real 1787763962]  req@ffff9751f87bc3c0 x1874599832677504/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763962.942159:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763962.942160:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763962.942162:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763962.942163:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763962.942165:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763962.942166:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763962.942169:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc3c0 x1874599832677504/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763962.942174:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc3c0 x1874599832677504/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.942179:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763962.942180:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.942181:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763962.942182:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763962.942184:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc3c0 x1874599832677504/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.942188:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763962.942191:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc3c0 x1874599832677504/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.942196:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763962.942197:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.942198:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763962.942199:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.942200:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763962.942201:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763962.942203:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763962.942204:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763962.942206:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763962.942208:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc3c0 x1874599832677504/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763962.942217:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832677504:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763962.942220:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763962.942223:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc780 x1874599832677632/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.942226:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763962.942227:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763962.942229:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763962.942230:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763962.942231:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763962.942233:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832677632:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763962.942234:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763962.942236:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763962.942237:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763962.942238:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763962.942241:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763962.942242:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763962.942244:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763962.942246:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763962.942248:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763962.942251:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832677632, portal 4
00000100:00000001:2.0:1787763962.942252:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763962.942253:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049847680 : -115096659703936 : ffff9751f87bc780)
00000100:00000040:2.0:1787763962.942257:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc780 x1874599832677632/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.942262:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763962.942263:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763962.942265:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763962.942266:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832677632, offset 0
00000400:00000200:2.0:1787763962.942268:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763962.942271:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763962.942274:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763962.942276:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763962.942278:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763962.942280:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763962.942282:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763962.942284:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58529:58529:256:4294967295] 192.168.203.110@tcp LPNI seq info [58529:58529:8:4294967295]
00000400:00000200:2.0:1787763962.942287:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763962.942291:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763962.942294:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763962.942296:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763962.942299:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763962.942301:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787763962.942304:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763962.942305:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787763962.942308:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763962.942310:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763962.942312:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763962.942314:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763962.942316:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763962.942317:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763962.942319:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763962.942321:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc780 x1874599832677632/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.942326:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763962.942328:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc780 x1874599832677632/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763987 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.942333:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763962.942334:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763962.942336:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763962.942337:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787763962.942372:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763962.942383:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316349990
00000100:00000001:2.0:1787763962.942393:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316351220
00000100:00000001:2.0:1787763962.942394:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316349990
00000100:00000001:2.0:1787763962.942396:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316351220
00000100:00000001:2.0:1787763962.942406:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763962.942407:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763962.942409:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763962.942411:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763962/real 1787763962]  req@ffff9751f87bc780 x1874599832677632/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763962.942416:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763962.942417:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763962.942418:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763962.942420:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763962.942421:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763962.942422:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763962.942424:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc780 x1874599832677632/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763962.942430:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc780 x1874599832677632/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.942450:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763962.942452:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.942453:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763962.942461:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763962.942464:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc780 x1874599832677632/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.942468:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763962.942471:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc780 x1874599832677632/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.942475:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763962.942477:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.942478:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763962.942479:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.942481:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763962.942482:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763962.942498:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763962.942501:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763962.942503:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763962.942506:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc780 x1874599832677632/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763962.942512:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832677632:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763962.942514:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.942516:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763962.942518:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bfc00 x1874599832677376/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.942522:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763962.942523:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763962.942525:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763962.942526:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763962.942527:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763962.942528:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763962.942530:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763962.942531:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612800.
02000000:00000001:2.0:1787763962.942534:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763962.942535:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763962.942536:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bfc00.
00000100:00000001:2.0:1787763962.942538:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763962.942539:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.942541:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763962.942543:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc3c0 x1874599832677504/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.942551:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763962.942552:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763962.942553:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763962.942555:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763962.942556:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763962.942557:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763962.942558:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763962.942559:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612c00.
02000000:00000001:2.0:1787763962.942560:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763962.942560:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763962.942561:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc3c0.
00000100:00000001:2.0:1787763962.942563:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763962.942563:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.942564:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763962.942566:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc780 x1874599832677632/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763987 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763962.942570:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763962.942570:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763962.942571:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763962.942572:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763962.942573:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763962.942574:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787763962.942575:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763962.942576:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612400.
02000000:00000001:2.0:1787763962.942577:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763962.942577:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763962.942578:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc780.
00000100:00000001:2.0:1787763962.942579:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763962.942580:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.942581:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763962.942583:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763962.942584:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763962.942585:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763962.942586:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763963.580306:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763963.580307:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763963.580315:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763963.580315:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:1.0:1787763963.580319:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763963.580320:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763963.580324:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763963.580325:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763963.580328:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763963.580331:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763963.580332:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763963.580333:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763963.580336:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763963.580337:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763963.580344:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763963.580345:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763963.580347:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763963.580348:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763963.580351:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763963.580353:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763963.580356:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763963.580410:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763963.580416:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763963.580420:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763963.964273:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763963.964284:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763963.964286:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763963.964293:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763963.964297:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763963.964302:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763963.964305:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763963.964309:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763963.964313:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763964.220279:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316351220, at 4316351232
00000100:00000010:3.0:1787763964.220294:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7360.
00000100:00000040:2.0:1787763964.220367:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316351220, at 4316351232
00000100:00000010:2.0:1787763964.220383:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00000100:00000001:1.0:1787763964.604250:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763964.604257:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763964.604259:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763964.604263:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787763964.604264:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763964.604270:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763964.604270:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763964.604273:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763964.604277:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763964.604279:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763964.604280:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763964.604284:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763964.604285:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763964.604294:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763964.604296:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763964.604391:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763964.604398:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763964.604402:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763964.988266:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763964.988275:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763964.988277:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763964.988282:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763964.988286:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763964.988290:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763964.988304:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763964.988308:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763964.988311:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763965.628277:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763965.628286:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763965.628288:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763965.628294:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763965.628294:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763965.628304:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763965.628305:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763965.628316:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763965.628317:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763965.628319:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763965.628328:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763965.628329:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763965.628336:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763965.628338:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763965.628345:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763965.628404:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763965.628408:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763965.628411:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763966.012236:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763966.012245:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763966.012254:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763966.012260:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763966.012264:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763966.012267:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763966.012269:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763966.012272:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763966.012275:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763966.652240:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763966.652249:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763966.652253:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763966.652257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787763966.652258:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763966.652265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763966.652267:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763966.652270:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763966.652273:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763966.652274:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763966.652275:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763966.652280:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763966.652281:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763966.652286:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763966.652290:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763966.652293:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763966.652296:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763966.652298:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763966.652301:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763966.652305:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763966.652307:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763966.652361:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763966.652366:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763966.652371:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763967.036225:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763967.036233:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763967.036235:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763967.036240:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763967.036244:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763967.036247:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763967.036250:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763967.036253:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763967.036255:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763967.676281:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763967.676284:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763967.676292:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763967.676295:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763967.676302:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763967.676303:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763967.676308:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763967.676310:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763967.676313:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763967.676316:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763967.676316:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763967.676325:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763967.676329:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763967.676331:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763967.676333:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763967.676371:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763967.676375:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763967.676378:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763968.060209:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763968.060222:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763968.060225:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763968.060232:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1787763968.060235:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763968.060244:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1787763968.060245:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787763968.060256:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787763968.060257:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763968.060268:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:3.0:1787763968.060269:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763968.060276:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763968.060278:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763968.060287:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763968.060292:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21682
00000400:00000200:3.0:1787763968.060300:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763968.060307:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763968.060309:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763968.060312:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763968.060315:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763968.060317:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763968.060320:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763968.060323:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763968.060324:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787763968.060327:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763968.060330:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787763968.060332:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787763968.060333:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787763968.060335:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763968.060336:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763968.060338:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763968.060339:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763968.060341:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787763968.060343:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763968.060345:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763968.060348:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763968.060351:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763968.060353:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763968.060355:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787763968.060357:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763968.060364:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bcf00.
00000020:00000040:3.0:1787763968.060367:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787763968.060369:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763968.060371:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763968.060373:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763968.060375:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763968.060379:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763968.060385:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613800.
00000100:00000001:3.0:1787763968.060390:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763968.060397:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bcf00 x1874599832677760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787763968.060406:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787763968.060409:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763968.060413:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ebc1bc5] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bcf00 x1874599832677760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763968.060433:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763968.060435:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787763968.060438:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787763968.060439:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763968.060440:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763968.060442:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787763968.060446:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21682
00000400:00000200:3.0:1787763968.060461:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763968.060464:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763968.060466:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763968.060468:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763968.060470:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763968.060471:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763968.060484:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1787763968.060484:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763968.060492:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1787763968.060492:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763968.060498:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763968.060501:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:3.0:1787763968.060503:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763968.060508:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000040:2.0:1787763968.060509:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bcf00 x1874599832677760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787763968.060514:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:3.0:1787763968.060521:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:3.0:1787763968.060536:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763968.060542:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763968.060544:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763968.060549:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763968.060553:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:2.0:1787763968.060557:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1787763968.060558:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:2.0:1787763968.060567:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:3.0:1787763968.060567:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:2.0:1787763968.060578:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763968.060579:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763968.060586:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763968.060588:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832677760:192.168.203.110@tcp:250:kworker.0
02000000:00000001:3.0:1787763968.060592:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787763968.060597:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763968.060600:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763968.060602:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763968.060604:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763968.060606:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763968.060608:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1787763968.060611:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:3.0:1787763968.060613:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787763968.060618:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763968.060620:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
00000100:00000001:3.0:1787763968.060621:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763968.060625:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763968.060629:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc780.
00000020:00000040:3.0:1787763968.060632:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000200:2.0:1787763968.060639:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:3.0:1787763968.060639:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:2.0:1787763968.060647:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
02000000:00000001:3.0:1787763968.060647:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763968.060654:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763968.060656:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:2.0:1787763968.060657:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
02000000:00000001:3.0:1787763968.060659:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763968.060667:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832677760, portal 25
02000000:00000010:3.0:1787763968.060668:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613c00.
00000100:00000001:2.0:1787763968.060671:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763968.060674:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049849600 : -115096659702016 : ffff9751f87bcf00)
00000100:00000001:3.0:1787763968.060676:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763968.060682:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bcf00 x1874599832677760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763968.060692:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc780 x1874599832677888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763968.060725:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1787763968.060732:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:2.0:1787763968.060735:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787763968.060738:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763968.060739:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763968.060750:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832677760, offset 0
00000100:00000040:3.0:1787763968.060754:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7d56184] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc780 x1874599832677888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787763968.060767:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787763968.060767:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763968.060782:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:3.0:1787763968.060782:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787763968.060788:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1787763968.060790:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763968.060794:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763968.060797:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:2.0:1787763968.060800:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00080000:3.0:1787763968.060805:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21682
00000400:00000200:2.0:1787763968.060814:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763968.060819:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763968.060823:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763968.060825:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763968.060829:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58530:58530:256:4294967295] 192.168.203.110@tcp LPNI seq info [58530:58530:8:4294967295]
00000400:00000200:2.0:1787763968.060833:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763968.060843:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763968.060851:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763968.060853:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763968.060862:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000800:00000010:2.0:1787763968.060863:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000100:00080000:3.0:1787763968.060865:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763968.060870:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:2.0:1787763968.060877:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787763968.060878:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1787763968.060888:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000100:00000040:3.0:1787763968.060889:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763968.060901:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763968.060910:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:2.1:1787763968.060910:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:3.0:1787763968.060917:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:3.0:1787763968.060922:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:2.1:1787763968.060922:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000020:00000040:3.0:1787763968.060927:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:3.0:1787763968.060929:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000100:2.0:1787763968.060934:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000020:00000001:3.0:1787763968.060936:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000010:2.0:1787763968.060937:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000100:00000040:3.0:1787763968.060944:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763968.060947:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763968.060951:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787763968.060954:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1787763968.060954:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763968.060967:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:2.0:1787763968.060973:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:3.0:1787763968.060976:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:2.0:1787763968.060981:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000100:00000001:3.0:1787763968.060981:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763968.060988:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
02000000:00000001:3.0:1787763968.060988:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763968.060993:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763968.060996:0:257575:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:3.0:1787763968.060997:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763968.061003:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000200:2.0:1787763968.061009:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bcf00 x1874599832677760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:3.0:1787763968.061009:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787763968.061020:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763968.061029:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763968.061034:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000010:3.0:1787763968.061035:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc3c0.
00000020:00000040:3.0:1787763968.061039:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000040:2.0:1787763968.061041:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bcf00 x1874599832677760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763993 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787763968.061044:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763968.061046:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763968.061054:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763968.061059:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763968.061063:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763968.061074:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611800.
00000100:00000001:2.0:1787763968.061078:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763968.061080:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763968.061081:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763968.061092:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763968.061092:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc3c0 x1874599832678016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787763968.061109:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763968.061117:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316356339
00000100:00000001:3.0:1787763968.061121:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787763968.061123:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763968.061126:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00000040:3.0:1787763968.061132:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000060bf8470] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc3c0 x1874599832678016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763968.061160:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763968.061162:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763968.061165:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000040:3.0:1787763968.061166:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:3.0:1787763968.061173:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787763968.061177:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787763968.061180:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763968.061182:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763968.061183:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316356340
00000100:00000001:3.0:1787763968.061186:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787763968.061190:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21682
00000100:00000010:2.0:1787763968.061191:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7a80.
00000400:00000200:3.0:1787763968.061195:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763968.061201:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000400:00000200:3.0:1787763968.061201:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763968.061207:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763968.061213:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763968.061214:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763968.061215:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763968.061221:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763968.061224:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763968.061225:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763968.061236:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000020:00000001:3.0:1787763968.061236:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763968.061243:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:3.0:1787763968.061246:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00100000:2.0:1787763968.061252:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763968/real 1787763968]  req@ffff9751f87bcf00 x1874599832677760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787763968.061252:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:3.0:1787763968.061259:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:3.0:1787763968.061269:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000001:2.0:1787763968.061284:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:3.0:1787763968.061285:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000001:2.0:1787763968.061287:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:3.0:1787763968.061288:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1787763968.061291:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000100:00000040:3.0:1787763968.061295:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787763968.061297:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000100:00000001:3.0:1787763968.061298:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000010:2.0:1787763968.061304:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000020:00000040:3.0:1787763968.061305:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:2.0:1787763968.061310:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00080000:3.0:1787763968.061311:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763968.061319:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763968.061323:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763968.061326:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763968.061327:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763968.061328:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000200:2.0:1787763968.061329:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bcf00 x1874599832677760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:3.0:1787763968.061331:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1787763968.061343:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763968.061353:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763968.061364:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bfc00.
00000100:00000200:2.0:1787763968.061371:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bcf00 x1874599832677760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787763968.061371:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:3.0:1787763968.061381:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763968.061386:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763968.061398:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1787763968.061400:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000001:3.0:1787763968.061401:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787763968.061403:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787763968.061404:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763968.061418:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000010:3.0:1787763968.061420:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610400.
00000100:00000001:2.0:1787763968.061421:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763968.061430:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763968.061433:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bcf00 x1874599832677760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763968.061440:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bfc00 x1874599832678144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763968.061470:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763968.061480:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:2.0:1787763968.061485:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bcf00 x1874599832677760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763968.061485:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763968.061500:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003c1f6b31] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bfc00 x1874599832678144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.061507:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763968.061520:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763968.061529:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763968.061536:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763968.061538:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763968.061543:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763968.061546:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:3.0:1787763968.061546:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21692)
00000100:00080000:2.0:1787763968.061554:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763968.061558:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763968.061562:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763968.061568:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bcf00 x1874599832677760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763968.061575:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832677760:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787763968.061578:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763968.061580:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763968.061582:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bcf00 x1874599832677760/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.061586:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763968.061589:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763968.061591:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763968.061594:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763968.061596:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763968.061597:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787763968.061599:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763968.061601:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613800.
02000000:00000001:2.0:1787763968.061605:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763968.061607:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763968.061609:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bcf00.
00000100:00000001:2.0:1787763968.061612:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763968.061612:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763968.061614:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763968.061621:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763968.061622:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763968.061623:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763968.061625:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763968.061626:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763968.061628:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc780 x1874599832677888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.061634:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763968.061635:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763968.061637:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763968.061639:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763968.061640:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763968.061642:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832677888:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763968.061644:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763968.061646:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763968.061648:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763968.061649:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763968.061651:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763968.061653:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763968.061656:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763968.061658:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763968.061661:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763968.061664:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832677888, portal 10
00000100:00000001:2.0:1787763968.061665:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763968.061666:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049847680 : -115096659703936 : ffff9751f87bc780)
00000100:00000040:2.0:1787763968.061673:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc780 x1874599832677888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.061678:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763968.061679:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763968.061681:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763968.061683:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832677888, offset 0
00000400:00000200:2.0:1787763968.061686:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763968.061689:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763968.061692:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763968.061695:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763968.061697:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763968.061699:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763968.061701:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763968.061703:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58531:58531:256:4294967295] 192.168.203.110@tcp LPNI seq info [58531:58531:8:4294967295]
00000400:00000200:2.0:1787763968.061706:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763968.061710:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763968.061714:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763968.061716:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763968.061719:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763968.061721:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787763968.061725:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763968.061726:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1787763968.061729:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763968.061731:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763968.061733:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763968.061735:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763968.061737:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763968.061739:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763968.061740:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763968.061743:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc780 x1874599832677888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.061748:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763968.061751:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc780 x1874599832677888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763993 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.061755:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763968.061756:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763968.061759:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763968.061763:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787763968.061765:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763968.061767:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316355104
00000100:00000001:2.0:1787763968.061769:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316356339
00000100:00000001:2.0:1787763968.061770:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316355104
00000100:00000001:2.0:1787763968.061772:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316356340
00000100:00000001:2.0:1787763968.061773:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763968.061774:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763968.061775:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763968.061778:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763968/real 1787763968]  req@ffff9751f87bc780 x1874599832677888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763968.061783:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763968.061784:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763968.061785:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763968.061787:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763968.061788:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763968.061789:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763968.061791:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc780 x1874599832677888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763968.061796:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc780 x1874599832677888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.061801:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763968.061802:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763968.061803:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763968.061804:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763968.061806:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc780 x1874599832677888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.061810:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763968.061812:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc780 x1874599832677888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.061818:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763968.061820:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763968.061821:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763968.061822:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763968.061823:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763968.061833:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763968.061834:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763968.061836:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763968.061837:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763968.061840:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc780 x1874599832677888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763968.061845:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832677888:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763968.061848:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763968.061850:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc3c0 x1874599832678016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.061854:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763968.061855:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763968.061857:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763968.061858:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763968.061859:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763968.061861:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832678016:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763968.061863:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763968.061864:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763968.061865:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763968.061866:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763968.061869:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763968.061870:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763968.061872:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763968.061874:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763968.061876:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763968.061878:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832678016, portal 10
00000100:00000001:2.0:1787763968.061879:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763968.061880:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049846720 : -115096659704896 : ffff9751f87bc3c0)
00000100:00000040:2.0:1787763968.061884:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc3c0 x1874599832678016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.061889:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763968.061890:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763968.061892:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763968.061893:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832678016, offset 0
00000400:00000200:2.0:1787763968.061896:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763968.061898:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763968.061900:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763968.061901:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763968.061903:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763968.061905:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763968.061906:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763968.061909:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58532:58532:256:4294967295] 192.168.203.110@tcp LPNI seq info [58532:58532:8:4294967295]
00000400:00000200:2.0:1787763968.061911:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763968.061916:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763968.061918:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763968.061920:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763968.061922:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763968.061924:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787763968.061927:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763968.061929:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1787763968.061931:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763968.061933:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763968.061934:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763968.061937:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763968.061939:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763968.061941:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763968.061942:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763968.061944:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc3c0 x1874599832678016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.061950:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763968.061951:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc3c0 x1874599832678016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763993 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.061956:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763968.061957:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763968.061959:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763968.061961:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787763968.061963:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763968.061964:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316355092
00000100:00000001:2.0:1787763968.061966:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316356339
00000100:00000001:2.0:1787763968.061968:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316355092
00000100:00000001:2.0:1787763968.061969:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316356340
00000100:00000001:2.0:1787763968.061970:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763968.061971:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763968.061973:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763968.061975:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763968/real 1787763968]  req@ffff9751f87bc3c0 x1874599832678016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763968.061979:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763968.061980:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763968.061981:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763968.061982:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763968.061983:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763968.061984:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763968.061986:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc3c0 x1874599832678016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763968.061992:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc3c0 x1874599832678016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.061997:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763968.061998:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763968.061999:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763968.062000:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763968.062003:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc3c0 x1874599832678016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.062007:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763968.062009:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc3c0 x1874599832678016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.062013:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763968.062014:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763968.062015:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763968.062016:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763968.062017:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763968.062018:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763968.062019:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763968.062021:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763968.062022:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763968.062025:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc3c0 x1874599832678016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763968.062030:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832678016:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763968.062032:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763968.062034:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bfc00 x1874599832678144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.062039:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763968.062040:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763968.062042:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763968.062043:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763968.062045:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763968.062047:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832678144:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763968.062049:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763968.062050:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763968.062051:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763968.062052:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763968.062054:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763968.062055:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763968.062058:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763968.062060:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763968.062062:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763968.062064:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832678144, portal 4
00000100:00000001:2.0:1787763968.062065:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763968.062066:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049861120 : -115096659690496 : ffff9751f87bfc00)
00000100:00000040:2.0:1787763968.062070:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bfc00 x1874599832678144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.062075:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763968.062076:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763968.062078:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763968.062079:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832678144, offset 0
00000400:00000200:2.0:1787763968.062081:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763968.062083:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763968.062085:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763968.062087:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763968.062088:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763968.062090:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763968.062091:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763968.062094:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58533:58533:256:4294967295] 192.168.203.110@tcp LPNI seq info [58533:58533:8:4294967295]
00000400:00000200:2.0:1787763968.062096:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763968.062101:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763968.062104:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763968.062106:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763968.062108:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763968.062110:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787763968.062112:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763968.062114:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1787763968.062117:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763968.062118:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763968.062120:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763968.062122:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763968.062123:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763968.062124:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763968.062126:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763968.062128:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bfc00 x1874599832678144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.062150:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763968.062153:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bfc00 x1874599832678144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763993 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.062157:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763968.062159:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763968.062161:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763968.062163:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787763968.062165:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763968.062167:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316355110
00000100:00000001:2.0:1787763968.062169:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316356339
00000100:00000001:2.0:1787763968.062170:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316355110
00000100:00000001:2.0:1787763968.062172:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316356340
00000100:00000001:2.0:1787763968.062173:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763968.062174:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763968.062175:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763968.062178:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763968/real 1787763968]  req@ffff9751f87bfc00 x1874599832678144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763968.062186:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763968.062188:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763968.062190:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763968.062193:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763968.062194:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763968.062196:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763968.062199:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bfc00 x1874599832678144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763968.062209:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bfc00 x1874599832678144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.062216:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763968.062218:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763968.062220:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763968.062221:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763968.062226:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bfc00 x1874599832678144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.062234:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763968.062238:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bfc00 x1874599832678144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.062243:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763968.062245:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763968.062246:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763968.062247:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763968.062248:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763968.062250:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763968.062251:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763968.062253:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763968.062255:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763968.062258:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bfc00 x1874599832678144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763968.062264:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832678144:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763968.062266:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763968.062267:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763968.062269:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc780 x1874599832677888/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.062273:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763968.062274:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763968.062276:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763968.062277:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763968.062278:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763968.062279:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763968.062281:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763968.062282:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613c00.
02000000:00000001:2.0:1787763968.062284:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763968.062285:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763968.062287:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc780.
00000100:00000001:2.0:1787763968.062288:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763968.062289:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763968.062290:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763968.062292:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc3c0 x1874599832678016/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.062296:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763968.062297:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763968.062298:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763968.062299:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763968.062300:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763968.062300:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763968.062301:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763968.062302:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611800.
02000000:00000001:2.0:1787763968.062304:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763968.062306:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763968.062307:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc3c0.
00000100:00000001:2.0:1787763968.062308:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763968.062309:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763968.062310:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763968.062312:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bfc00 x1874599832678144/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763993 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763968.062316:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763968.062317:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763968.062318:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763968.062319:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763968.062320:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763968.062321:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787763968.062322:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763968.062322:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610400.
02000000:00000001:2.0:1787763968.062324:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763968.062324:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763968.062325:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bfc00.
00000100:00000001:2.0:1787763968.062327:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763968.062327:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763968.062329:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763968.062331:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763968.062332:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763968.062333:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763968.062334:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763968.700262:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763968.700268:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763968.700270:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763968.700271:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763968.700276:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763968.700277:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763968.700282:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763968.700282:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763968.700287:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763968.700289:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763968.700293:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763968.700294:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763968.700299:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763968.700299:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763968.700306:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763968.700354:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763968.700359:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763968.700361:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763969.084260:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763969.084271:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763969.084274:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763969.084282:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763969.084286:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763969.084293:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763969.084298:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763969.084301:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763969.084307:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763969.340288:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316356340, at 4316356352
00000100:00000010:3.0:1787763969.340315:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7a80.
00000100:00000040:2.0:1787763969.340347:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316356339, at 4316356352
00000100:00000010:2.0:1787763969.340375:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00000100:00000001:1.0:1787763969.724236:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763969.724243:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763969.724245:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763969.724246:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787763969.724249:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763969.724252:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763969.724254:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763969.724258:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763969.724262:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763969.724262:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763969.724265:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763969.724265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763969.724269:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763969.724271:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763969.724274:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763969.724333:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763969.724338:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763969.724342:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763970.108215:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763970.108222:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763970.108224:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763970.108228:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763970.108232:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763970.108235:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763970.108236:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763970.108238:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763970.108240:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763970.748223:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763970.748229:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763970.748231:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763970.748234:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763970.748238:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763970.748238:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763970.748242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763970.748243:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763970.748246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763970.748249:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763970.748255:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763970.748258:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763970.748261:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763970.748280:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763970.748283:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763970.748312:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763970.748317:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763970.748321:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763971.132250:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763971.132256:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763971.132258:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763971.132262:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763971.132266:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763971.132268:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763971.132270:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763971.132273:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763971.132275:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763971.772227:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763971.772231:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763971.772235:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763971.772235:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763971.772239:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763971.772241:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787763971.772242:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763971.772242:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763971.772247:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763971.772247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763971.772251:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763971.772251:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:1.0:1787763971.772254:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763971.772254:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:1.0:1787763971.772258:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763971.772258:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763971.772261:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:1.0:1787763971.772262:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763971.772264:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763971.772266:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763971.772269:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763971.772316:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763971.772320:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763971.772323:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763972.156218:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763972.156227:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763972.156230:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763972.156236:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763972.156240:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763972.156243:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763972.156245:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763972.156248:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763972.156250:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763972.796219:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763972.796220:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763972.796225:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763972.796228:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763972.796231:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763972.796233:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763972.796234:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763972.796240:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763972.796241:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763972.796245:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763972.796245:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763972.796249:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763972.796252:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763972.796254:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763972.796257:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763972.796295:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763972.796299:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763972.796303:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763973.180233:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1787763973.180233:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763973.180241:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:3.0:1787763973.180242:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787763973.180245:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763973.180245:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763973.180249:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1787763973.180251:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787763973.180254:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763973.180257:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763973.180262:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1787763973.180262:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21687
00000100:00000001:2.0:1787763973.180267:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763973.180271:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:3.0:1787763973.180272:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763973.180278:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763973.180281:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763973.180284:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763973.180287:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763973.180289:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763973.180292:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763973.180294:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763973.180297:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763973.180299:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787763973.180301:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763973.180303:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787763973.180305:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787763973.180306:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787763973.180308:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763973.180310:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763973.180311:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763973.180312:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763973.180314:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787763973.180316:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787763973.180318:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763973.180320:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763973.180322:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763973.180324:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763973.180327:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787763973.180328:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787763973.180336:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bde00.
00000020:00000040:3.0:1787763973.180339:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787763973.180342:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787763973.180343:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763973.180345:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763973.180347:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763973.180351:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763973.180357:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611c00.
00000100:00000001:3.0:1787763973.180362:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763973.180369:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bde00 x1874599832678272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787763973.180377:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787763973.180381:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763973.180384:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000941163c2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bde00 x1874599832678272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763973.180402:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787763973.180404:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787763973.180407:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787763973.180408:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763973.180409:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763973.180411:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787763973.180426:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21687
00000400:00000200:3.0:1787763973.180429:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763973.180431:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:3.0:1787763973.180432:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787763973.180436:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1787763973.180436:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787763973.180440:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763973.180441:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763973.180444:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787763973.180448:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000040:2.0:1787763973.180449:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bde00 x1874599832678272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763973.180450:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763973.180456:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763973.180458:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:3.0:1787763973.180461:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1787763973.180464:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:3.0:1787763973.180465:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000001:2.0:1787763973.180466:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787763973.180469:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:2.0:1787763973.180471:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:3.0:1787763973.180472:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:2.0:1787763973.180474:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1787763973.180477:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:2.0:1787763973.180478:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763973.180482:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:2.0:1787763973.180485:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832678272:192.168.203.110@tcp:250:kworker.0
00000100:00000040:3.0:1787763973.180488:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763973.180491:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1787763973.180492:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787763973.180495:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:2.0:1787763973.180496:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:3.0:1787763973.180498:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:2.0:1787763973.180500:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763973.180502:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763973.180503:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:3.0:1787763973.180505:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:2.0:1787763973.180509:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:3.0:1787763973.180511:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763973.180513:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763973.180514:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763973.180517:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787763973.180521:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:2.0:1787763973.180522:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000008:00000001:3.0:1787763973.180524:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763973.180527:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763973.180528:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000100:00000010:3.0:1787763973.180531:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bed00.
00000020:00000040:3.0:1787763973.180534:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787763973.180537:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:2.0:1787763973.180539:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
02000000:00000001:3.0:1787763973.180539:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763973.180545:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763973.180547:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:2.0:1787763973.180549:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832678272, portal 25
02000000:00000001:3.0:1787763973.180550:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763973.180556:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000010:3.0:1787763973.180559:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610c00.
00000100:00000001:2.0:1787763973.180560:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049853440 : -115096659698176 : ffff9751f87bde00)
00000100:00000001:3.0:1787763973.180568:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763973.180571:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bde00 x1874599832678272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.180581:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763973.180584:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763973.180587:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763973.180590:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832678272, offset 0
00000100:00100000:3.0:1787763973.180590:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bed00 x1874599832678400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787763973.180599:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787763973.180606:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1787763973.180609:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763973.180610:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763973.180617:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000079751c9a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bed00 x1874599832678400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787763973.180618:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763973.180625:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1787763973.180632:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763973.180633:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00080000:3.0:1787763973.180637:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787763973.180638:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763973.180644:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00080000:3.0:1787763973.180644:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:2.0:1787763973.180649:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58534:58534:256:4294967295] 192.168.203.110@tcp LPNI seq info [58534:58534:8:4294967295]
00000100:00000001:3.0:1787763973.180649:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787763973.180651:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787763973.180657:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:2.0:1787763973.180658:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:3.0:1787763973.180661:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21687
00000400:00000200:2.0:1787763973.180667:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763973.180674:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763973.180675:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000010:2.0:1787763973.180681:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000400:00000200:3.0:1787763973.180681:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763973.180685:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787763973.180689:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:2.0:1787763973.180692:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787763973.180693:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1787763973.180697:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000100:00000040:3.0:1787763973.180698:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763973.180703:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787763973.180706:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:2.1:1787763973.180707:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:3.0:1787763973.180708:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:3.0:1787763973.180711:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763973.180714:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000200:2.1:1787763973.180714:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:3.0:1787763973.180720:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000100:2.0:1787763973.180723:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000020:00000001:3.0:1787763973.180724:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000010:2.0:1787763973.180726:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000100:00000040:3.0:1787763973.180730:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763973.180732:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1787763973.180734:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1787763973.180735:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763973.180739:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787763973.180740:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000020:00000040:3.0:1787763973.180743:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:2.0:1787763973.180746:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000100:00080000:3.0:1787763973.180748:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000010:2.0:1787763973.180750:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:3.0:1787763973.180753:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763973.180755:0:257575:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:3.0:1787763973.180756:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787763973.180760:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:2.0:1787763973.180761:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bde00 x1874599832678272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787763973.180764:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763973.180769:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787763973.180771:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787763973.180774:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763973.180775:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1787763973.180777:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763973.180780:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bde00 x1874599832678272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787763998 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1787763973.180783:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87be1c0.
00000020:00000040:3.0:1787763973.180788:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787763973.180793:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1787763973.180794:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763973.180795:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763973.180797:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763973.180798:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763973.180800:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787763973.180802:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787763973.180804:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610800.
00000100:00000001:3.0:1787763973.180807:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763973.180811:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763973.180812:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87be1c0 x1874599832678528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787763973.180817:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763973.180821:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316361459
00000100:00000001:3.0:1787763973.180825:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787763973.180828:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763973.180829:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00000040:3.0:1787763973.180833:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e2011c6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87be1c0 x1874599832678528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.180836:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787763973.180842:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763973.180844:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763973.180845:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316361459
00000100:00000040:3.0:1787763973.180847:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000010:2.0:1787763973.180849:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb74e0.
00000100:00080000:3.0:1787763973.180850:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763973.180853:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00080000:3.0:1787763973.180854:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787763973.180857:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787763973.180858:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:3.0:1787763973.180859:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763973.180862:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763973.180864:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763973.180865:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00080000:3.0:1787763973.180868:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21687
00000100:00100000:2.0:1787763973.180872:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763973/real 1787763973]  req@ffff9751f87bde00 x1874599832678272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787763973.180874:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763973.180880:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787763973.180886:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000001:2.0:1787763973.180887:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763973.180889:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:3.0:1787763973.180891:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000010:2.0:1787763973.180892:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000100:00000001:3.0:1787763973.180895:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1787763973.180896:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000100:00000040:3.0:1787763973.180900:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000010:2.0:1787763973.180901:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763973.180906:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:3.0:1787763973.180907:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000200:2.0:1787763973.180910:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bde00 x1874599832678272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787763973.180912:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787763973.180915:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:3.0:1787763973.180917:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787763973.180921:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000200:2.0:1787763973.180925:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bde00 x1874599832678272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787763973.180927:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:3.0:1787763973.180934:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:2.0:1787763973.180935:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763973.180938:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763973.180941:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763973.180943:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763973.180944:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00100000:2.0:1787763973.180948:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bde00 x1874599832678272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763973.180949:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763973.180958:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763973.180958:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787763973.180964:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:2.0:1787763973.180966:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bde00 x1874599832678272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787763973.180967:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:3.0:1787763973.180975:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763973.180981:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763973.180982:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763973.180987:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787763973.180987:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787763973.180991:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:3.0:1787763973.180992:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763973.180993:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763973.180997:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
02000000:00000001:3.0:1787763973.180998:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763973.180999:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763973.181001:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00080000:2.0:1787763973.181003:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000008:00100000:3.0:1787763973.181004:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00080000:2.0:1787763973.181008:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000008:00000001:3.0:1787763973.181009:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763973.181012:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:3.0:1787763973.181013:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763973.181018:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bde00 x1874599832678272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1787763973.181018:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc000.
00000020:00000040:3.0:1787763973.181024:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:3.0:1787763973.181030:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00100000:2.0:1787763973.181031:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832678272:192.168.203.110@tcp:250:kworker.0
02000000:00000001:3.0:1787763973.181032:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763973.181038:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763973.181038:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787763973.181039:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787763973.181041:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:3.0:1787763973.181043:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763973.181046:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bde00 x1874599832678272/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1787763973.181048:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612000.
00000100:00000001:3.0:1787763973.181052:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763973.181059:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
00000100:00100000:3.0:1787763973.181060:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc000 x1874599832678656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1787763973.181063:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763973.181066:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763973.181071:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000001:3.0:1787763973.181071:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000020:00000001:2.0:1787763973.181073:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000100:00000001:3.0:1787763973.181075:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1787763973.181076:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787763973.181080:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
00000100:00000040:3.0:1787763973.181081:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3fc1c6c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc000 x1874599832678656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1787763973.181083:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611c00.
02000000:00000001:2.0:1787763973.181088:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763973.181089:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763973.181092:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bde00.
00000100:00000001:3.0:1787763973.181093:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763973.181096:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763973.181099:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763973.181099:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21697)
00000100:00000001:2.0:1787763973.181104:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763973.181110:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763973.181111:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763973.181113:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763973.181115:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763973.181116:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763973.181120:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bed00 x1874599832678400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181125:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763973.181126:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763973.181128:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763973.181129:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763973.181131:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763973.181151:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832678400:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763973.181154:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763973.181156:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763973.181157:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763973.181158:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763973.181161:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763973.181162:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763973.181164:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763973.181166:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763973.181168:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763973.181171:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832678400, portal 10
00000100:00000001:2.0:1787763973.181172:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763973.181173:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049857280 : -115096659694336 : ffff9751f87bed00)
00000100:00000040:2.0:1787763973.181177:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bed00 x1874599832678400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181182:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763973.181183:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763973.181185:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763973.181186:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832678400, offset 0
00000400:00000200:2.0:1787763973.181190:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763973.181194:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763973.181198:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763973.181201:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763973.181204:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763973.181207:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763973.181209:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763973.181212:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58535:58535:256:4294967295] 192.168.203.110@tcp LPNI seq info [58535:58535:8:4294967295]
00000400:00000200:2.0:1787763973.181217:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763973.181224:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763973.181229:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763973.181234:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763973.181237:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763973.181240:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787763973.181245:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763973.181247:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1787763973.181252:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763973.181254:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763973.181257:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763973.181260:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763973.181264:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763973.181266:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763973.181269:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763973.181273:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bed00 x1874599832678400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181282:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763973.181285:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bed00 x1874599832678400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763998 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181290:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763973.181291:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763973.181294:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763973.181297:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787763973.181300:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763973.181301:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316360224
00000100:00000001:2.0:1787763973.181303:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316361459
00000100:00000001:2.0:1787763973.181305:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316360224
00000100:00000001:2.0:1787763973.181306:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316361459
00000100:00000001:2.0:1787763973.181308:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763973.181309:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763973.181311:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763973.181313:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763973/real 1787763973]  req@ffff9751f87bed00 x1874599832678400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763973.181323:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763973.181324:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763973.181325:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763973.181327:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763973.181328:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763973.181329:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763973.181331:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bed00 x1874599832678400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763973.181337:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bed00 x1874599832678400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181340:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763973.181341:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763973.181342:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763973.181343:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763973.181345:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bed00 x1874599832678400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181349:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763973.181352:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bed00 x1874599832678400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181357:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763973.181358:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763973.181359:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763973.181360:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763973.181361:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763973.181371:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763973.181372:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763973.181374:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763973.181376:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763973.181379:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bed00 x1874599832678400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763973.181384:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832678400:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763973.181387:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763973.181389:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87be1c0 x1874599832678528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181393:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763973.181394:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763973.181395:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763973.181396:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763973.181397:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763973.181399:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832678528:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763973.181401:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763973.181402:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763973.181403:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763973.181404:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763973.181407:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763973.181408:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763973.181410:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763973.181425:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763973.181430:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763973.181432:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832678528, portal 10
00000100:00000001:2.0:1787763973.181434:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763973.181434:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049854400 : -115096659697216 : ffff9751f87be1c0)
00000100:00000040:2.0:1787763973.181439:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87be1c0 x1874599832678528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181444:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763973.181445:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763973.181446:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763973.181448:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832678528, offset 0
00000400:00000200:2.0:1787763973.181451:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763973.181454:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763973.181457:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763973.181459:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763973.181461:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763973.181462:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763973.181464:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763973.181466:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58536:58536:256:4294967295] 192.168.203.110@tcp LPNI seq info [58536:58536:8:4294967295]
00000400:00000200:2.0:1787763973.181469:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763973.181473:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763973.181476:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763973.181478:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763973.181480:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763973.181482:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787763973.181485:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763973.181487:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1787763973.181490:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763973.181491:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763973.181493:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763973.181495:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763973.181497:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763973.181499:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763973.181500:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763973.181502:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87be1c0 x1874599832678528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181507:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763973.181509:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87be1c0 x1874599832678528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787763998 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181514:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763973.181516:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763973.181519:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763973.181521:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787763973.181525:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763973.181527:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316360212
00000100:00000001:2.0:1787763973.181530:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316361459
00000100:00000001:2.0:1787763973.181532:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316360212
00000100:00000001:2.0:1787763973.181534:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316361459
00000100:00000001:2.0:1787763973.181537:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763973.181539:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763973.181541:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763973.181545:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763973/real 1787763973]  req@ffff9751f87be1c0 x1874599832678528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763973.181554:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763973.181555:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763973.181557:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763973.181560:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763973.181561:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763973.181564:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763973.181567:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87be1c0 x1874599832678528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763973.181574:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87be1c0 x1874599832678528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181578:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763973.181579:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763973.181581:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763973.181582:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763973.181584:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87be1c0 x1874599832678528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181589:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763973.181591:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87be1c0 x1874599832678528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181595:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763973.181597:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763973.181598:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763973.181599:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763973.181600:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763973.181601:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763973.181603:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763973.181605:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763973.181606:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763973.181609:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87be1c0 x1874599832678528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763973.181613:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832678528:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763973.181616:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763973.181618:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc000 x1874599832678656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181623:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763973.181624:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763973.181625:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763973.181626:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763973.181628:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763973.181629:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832678656:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763973.181631:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763973.181632:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763973.181634:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763973.181635:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763973.181637:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763973.181638:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763973.181641:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763973.181642:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763973.181644:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763973.181646:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832678656, portal 4
00000100:00000001:2.0:1787763973.181648:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763973.181649:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049845760 : -115096659705856 : ffff9751f87bc000)
00000100:00000040:2.0:1787763973.181653:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc000 x1874599832678656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181657:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763973.181659:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763973.181660:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763973.181662:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832678656, offset 0
00000400:00000200:2.0:1787763973.181664:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763973.181667:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763973.181669:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763973.181671:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763973.181673:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763973.181674:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763973.181676:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763973.181678:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58537:58537:256:4294967295] 192.168.203.110@tcp LPNI seq info [58537:58537:8:4294967295]
00000400:00000200:2.0:1787763973.181681:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763973.181685:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763973.181687:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763973.181689:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763973.181692:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763973.181694:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787763973.181696:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763973.181698:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1787763973.181701:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763973.181702:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763973.181704:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763973.181706:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763973.181708:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763973.181709:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763973.181711:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763973.181713:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc000 x1874599832678656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181722:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763973.181724:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc000 x1874599832678656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787763998 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181729:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763973.181729:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763973.181731:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763973.181733:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787763973.181735:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763973.181736:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316360229
00000100:00000001:2.0:1787763973.181737:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316361459
00000100:00000001:2.0:1787763973.181739:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316360229
00000100:00000001:2.0:1787763973.181740:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316361459
00000100:00000001:2.0:1787763973.181742:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763973.181743:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763973.181744:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763973.181746:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763973/real 1787763973]  req@ffff9751f87bc000 x1874599832678656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763973.181751:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763973.181752:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763973.181753:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763973.181754:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763973.181755:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763973.181756:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763973.181758:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc000 x1874599832678656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763973.181763:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc000 x1874599832678656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181767:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763973.181768:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763973.181769:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763973.181770:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763973.181773:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc000 x1874599832678656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181777:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763973.181779:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc000 x1874599832678656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181783:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763973.181784:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763973.181785:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763973.181786:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763973.181787:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763973.181788:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763973.181789:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763973.181791:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763973.181792:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763973.181795:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc000 x1874599832678656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763973.181799:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832678656:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763973.181801:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763973.181802:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763973.181804:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bed00 x1874599832678400/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181809:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763973.181810:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763973.181811:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763973.181813:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763973.181814:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763973.181815:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763973.181817:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763973.181818:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610c00.
02000000:00000001:2.0:1787763973.181820:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763973.181821:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763973.181822:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bed00.
00000100:00000001:2.0:1787763973.181823:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763973.181824:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763973.181825:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763973.181827:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87be1c0 x1874599832678528/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181831:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763973.181832:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763973.181833:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763973.181834:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763973.181835:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763973.181836:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763973.181837:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763973.181838:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610800.
02000000:00000001:2.0:1787763973.181839:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763973.181840:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763973.181840:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87be1c0.
00000100:00000001:2.0:1787763973.181841:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763973.181842:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763973.181843:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763973.181845:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc000 x1874599832678656/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787763998 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763973.181849:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763973.181849:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763973.181850:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763973.181851:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763973.181852:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763973.181853:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787763973.181854:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763973.181855:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612000.
02000000:00000001:2.0:1787763973.181856:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763973.181857:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763973.181858:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc000.
00000100:00000001:2.0:1787763973.181860:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763973.181860:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763973.181862:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763973.181864:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763973.181865:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763973.181866:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763973.181867:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763973.820224:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763973.820229:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763973.820249:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763973.820251:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:1.0:1787763973.820252:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763973.820254:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00100000:1.0:1787763973.820258:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763973.820258:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763973.820262:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:1.0:1787763973.820263:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763973.820265:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763973.820267:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763973.820267:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763973.820270:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763973.820270:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763973.820273:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763973.820275:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763973.820276:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763973.820277:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763973.820281:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763973.820282:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763973.820347:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763973.820352:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763973.820355:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763974.204220:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763974.204227:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763974.204229:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763974.204233:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763974.204237:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763974.204240:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763974.204242:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763974.204243:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763974.204245:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763974.460260:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316361459, at 4316361472
00000100:00000010:3.0:1787763974.460281:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb74e0.
00000100:00000040:2.0:1787763974.460305:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316361459, at 4316361472
00000100:00000010:2.0:1787763974.460318:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00010000:00000001:3.0:1787763974.844293:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763974.844301:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763974.844308:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763974.844309:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763974.844315:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763974.844316:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763974.844317:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763974.844324:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763974.844328:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763974.844333:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763974.844338:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763974.844342:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763974.844346:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763974.844350:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763974.844354:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763974.844374:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763974.844379:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763974.844383:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763975.228270:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763975.228278:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763975.228281:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763975.228287:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763975.228293:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763975.228297:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763975.228301:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763975.228304:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763975.228308:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763975.868255:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763975.868258:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763975.868264:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763975.868267:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763975.868270:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763975.868273:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763975.868273:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763975.868281:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787763975.868282:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763975.868283:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763975.868289:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763975.868294:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763975.868297:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763975.868300:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763975.868303:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763975.868351:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763975.868356:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763975.868360:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763976.252260:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763976.252266:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763976.252269:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763976.252277:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763976.252283:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763976.252288:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763976.252292:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763976.252296:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763976.252300:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763976.892255:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763976.892259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763976.892264:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763976.892264:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763976.892269:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763976.892271:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787763976.892276:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763976.892277:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763976.892280:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:1.0:1787763976.892284:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763976.892284:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763976.892290:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:1.0:1787763976.892291:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763976.892292:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763976.892298:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763976.892299:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763976.892300:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763976.892305:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763976.892305:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763976.892308:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763976.892312:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763976.892359:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763976.892363:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763976.892366:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763977.276224:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763977.276230:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763977.276234:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763977.276239:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763977.276243:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763977.276246:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763977.276248:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763977.276251:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763977.276254:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763977.916262:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763977.916274:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763977.916277:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763977.916286:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763977.916289:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763977.916292:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763977.916297:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763977.916299:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763977.916305:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763977.916307:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763977.916310:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763977.916311:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763977.916317:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763977.916320:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763977.916324:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763977.916367:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763977.916374:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763977.916395:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763978.300253:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763978.300254:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763978.300263:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1787763978.300267:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787763978.300268:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763978.300276:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:2.0:1787763978.300278:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:0.0:1787763978.300281:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763978.300285:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763978.300287:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763978.300290:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763978.300294:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:0.0:1787763978.300298:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21692
00000100:00000001:2.0:1787763978.300299:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763978.300313:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787763978.300319:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763978.300328:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763978.300332:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763978.300336:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763978.300339:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763978.300341:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763978.300343:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763978.300347:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763978.300349:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763978.300352:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763978.300355:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763978.300357:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763978.300358:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763978.300360:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763978.300361:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763978.300363:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763978.300364:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763978.300367:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763978.300370:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763978.300371:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763978.300374:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763978.300390:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763978.300393:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763978.300395:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763978.300397:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763978.300408:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000020:00000040:0.0:1787763978.300412:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763978.300416:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763978.300417:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763978.300421:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763978.300423:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763978.300428:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763978.300438:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787763978.300445:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763978.300455:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832678784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763978.300465:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763978.300469:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763978.300474:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832678784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763978.300493:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763978.300495:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763978.300498:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763978.300499:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763978.300501:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763978.300503:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763978.300505:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21692
00000400:00000200:0.0:1787763978.300509:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763978.300513:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763978.300515:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763978.300518:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763978.300519:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763978.300521:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1787763978.300522:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763978.300525:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763978.300526:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1787763978.300534:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1787763978.300534:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1787763978.300545:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763978.300548:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763978.300551:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763978.300556:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:2.0:1787763978.300557:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876de00 x1874599832678784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787763978.300563:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763978.300579:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763978.300588:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763978.300590:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763978.300593:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763978.300594:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763978.300606:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:2.0:1787763978.300606:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:0.0:1787763978.300612:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:2.0:1787763978.300612:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763978.300620:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763978.300623:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832678784:192.168.203.110@tcp:250:kworker.0
00000100:00080000:0.0:1787763978.300630:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763978.300633:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1787763978.300635:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763978.300637:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763978.300639:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763978.300641:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763978.300643:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763978.300644:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1787763978.300653:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763978.300656:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000010:2.0:1787763978.300660:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
00000008:00100000:0.0:1787763978.300661:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1787763978.300668:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1787763978.300672:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763978.300677:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763978.300678:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000010:0.0:1787763978.300687:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
00000400:00000010:2.0:1787763978.300688:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000020:00000040:0.0:1787763978.300702:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763978.300706:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:2.0:1787763978.300707:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
02000000:00000001:0.0:1787763978.300708:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763978.300713:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763978.300715:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:2.0:1787763978.300716:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832678784, portal 25
02000000:00000001:0.0:1787763978.300717:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763978.300722:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763978.300724:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049525760 : -115096660025856 : ffff9751f876de00)
02000000:00000010:0.0:1787763978.300725:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787763978.300736:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763978.300742:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876de00 x1874599832678784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764003 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763978.300748:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832678912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763978.300779:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1787763978.300781:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763978.300791:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763978.300791:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763978.300803:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000040:0.0:1787763978.300806:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832678912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763978.300810:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832678784, offset 0
00000100:00000001:0.0:1787763978.300827:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763978.300831:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00080000:0.0:1787763978.300837:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763978.300841:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763978.300843:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763978.300848:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1787763978.300848:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763978.300858:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763978.300863:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21692
00000400:00000200:2.0:1787763978.300863:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763978.300878:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787763978.300879:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763978.300888:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787763978.300892:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1787763978.300896:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00080000:0.0:1787763978.300902:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1787763978.300903:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763978.300911:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58538:58538:256:4294967295] 192.168.203.110@tcp LPNI seq info [58538:58538:8:4294967295]
00000100:00000040:0.0:1787763978.300918:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:2.0:1787763978.300919:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763978.300930:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1787763978.300937:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763978.300944:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000200:2.0:1787763978.300944:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763978.300953:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000010:2.0:1787763978.300955:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000020:00000001:0.0:1787763978.300963:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763978.300965:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763978.300971:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:2.0:1787763978.300972:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:0.0:1787763978.300974:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000200:2.0:1787763978.300977:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000020:00000001:0.0:1787763978.300978:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763978.300987:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:2.1:1787763978.300988:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763978.300992:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000100:00000040:0.0:1787763978.300999:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763978.301010:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000100:2.0:1787763978.301015:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763978.301021:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000100:00000040:0.0:1787763978.301022:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763978.301026:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787763978.301027:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:0.0:1787763978.301036:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:2.0:1787763978.301043:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:0.0:1787763978.301044:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763978.301050:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763978.301051:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
02000000:00000001:0.0:1787763978.301061:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:2.0:1787763978.301062:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
02000000:00000001:0.0:1787763978.301070:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763978.301072:0:257575:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:0.0:1787763978.301077:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763978.301081:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000200:2.0:1787763978.301082:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876de00 x1874599832678784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764003 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1787763978.301084:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763978.301094:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763978.301103:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763978.301114:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000100:00000001:2.0:1787763978.301116:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000040:0.0:1787763978.301121:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000040:2.0:1787763978.301126:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876de00 x1874599832678784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764003 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763978.301128:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763978.301158:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763978.301160:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763978.301162:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763978.301164:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763978.301168:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787763978.301171:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763978.301181:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832679040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763978.301183:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763978.301194:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763978.301197:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763978.301199:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763978.301203:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832679040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.301205:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763978.301223:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763978.301233:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316366580
00000100:00000001:0.0:1787763978.301234:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763978.301239:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000010:2.0:1787763978.301241:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00000040:0.0:1787763978.301247:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763978.301250:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763978.301252:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00080000:0.0:1787763978.301255:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763978.301257:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763978.301259:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763978.301262:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316366580
00000100:00000001:0.0:1787763978.301263:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763978.301271:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21692
00000100:00000010:2.0:1787763978.301271:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7cc0.
00000100:00000001:2.0:1787763978.301280:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000400:00000200:0.0:1787763978.301281:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763978.301289:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1787763978.301291:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787763978.301294:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763978.301296:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787763978.301297:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000040:0.0:1787763978.301301:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00100000:2.0:1787763978.301306:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763978/real 1787763978]  req@ffff9751f876de00 x1874599832678784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763978.301307:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763978.301316:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763978.301328:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763978.301339:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000001:2.0:1787763978.301341:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000040:0.0:1787763978.301343:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000400:00000001:2.0:1787763978.301344:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000020:00000001:0.0:1787763978.301351:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:2.0:1787763978.301352:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000020:00000040:0.0:1787763978.301359:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000400:00000200:2.0:1787763978.301360:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000020:00000001:0.0:1787763978.301371:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000010:2.0:1787763978.301373:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000020:00000001:0.0:1787763978.301399:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:2.0:1787763978.301401:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:0.0:1787763978.301405:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000200:2.0:1787763978.301408:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876de00 x1874599832678784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763978.301410:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763978.301418:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763978.301426:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763978.301433:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763978.301446:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000200:2.0:1787763978.301446:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876de00 x1874599832678784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763978.301454:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763978.301460:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763978.301466:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763978.301476:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763978.301486:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787763978.301490:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000008:00100000:0.0:1787763978.301491:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:2.0:1787763978.301493:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000008:00000001:0.0:1787763978.301496:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763978.301498:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787763978.301500:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763978.301501:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:0.0:1787763978.301506:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000100:00100000:2.0:1787763978.301510:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876de00 x1874599832678784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787763978.301512:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763978.301522:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763978.301528:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763978.301538:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763978.301543:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787763978.301543:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787763978.301550:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763978.301555:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876de00 x1874599832678784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1787763978.301556:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787763978.301563:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763978.301566:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763978.301569:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787763978.301573:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832679168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763978.301574:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763978.301578:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763978.301588:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763978.301594:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787763978.301599:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00080000:2.0:1787763978.301602:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:0.0:1787763978.301603:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763978.301618:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000040:0.0:1787763978.301619:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832679168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.301629:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763978.301642:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876de00 x1874599832678784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763978.301644:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763978.301654:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21703)
00000100:00100000:2.0:1787763978.301660:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832678784:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787763978.301665:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763978.301667:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763978.301669:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876de00 x1874599832678784/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.301674:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763978.301676:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763978.301679:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763978.301681:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763978.301683:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763978.301684:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787763978.301686:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763978.301688:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014400.
02000000:00000001:2.0:1787763978.301690:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763978.301692:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763978.301693:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876de00.
00000100:00000001:2.0:1787763978.301698:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763978.301699:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763978.301700:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763978.301705:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763978.301706:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763978.301708:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763978.301709:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763978.301710:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763978.301713:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876da40 x1874599832678912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.301718:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763978.301719:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763978.301721:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763978.301723:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763978.301724:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763978.301726:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832678912:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763978.301728:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763978.301730:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763978.301731:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763978.301733:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763978.301735:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763978.301737:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763978.301740:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763978.301742:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763978.301744:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763978.301747:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832678912, portal 10
00000100:00000001:2.0:1787763978.301749:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763978.301750:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049524800 : -115096660026816 : ffff9751f876da40)
00000100:00000040:2.0:1787763978.301755:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876da40 x1874599832678912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764003 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.301759:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763978.301761:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763978.301762:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763978.301764:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832678912, offset 0
00000400:00000200:2.0:1787763978.301766:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763978.301769:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763978.301772:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763978.301774:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763978.301776:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763978.301778:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763978.301780:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763978.301782:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58539:58539:256:4294967295] 192.168.203.110@tcp LPNI seq info [58539:58539:8:4294967295]
00000400:00000200:2.0:1787763978.301785:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763978.301789:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763978.301792:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763978.301795:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763978.301797:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763978.301799:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787763978.301803:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763978.301805:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:2.0:1787763978.301808:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763978.301809:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763978.301811:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763978.301813:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763978.301815:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763978.301817:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763978.301818:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763978.301820:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876da40 x1874599832678912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764003 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.301830:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763978.301832:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876da40 x1874599832678912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764003 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.301836:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763978.301837:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763978.301839:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763978.301841:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787763978.301843:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763978.301845:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316365344
00000100:00000001:2.0:1787763978.301846:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316366580
00000100:00000001:2.0:1787763978.301848:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316365344
00000100:00000001:2.0:1787763978.301849:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316366580
00000100:00000001:2.0:1787763978.301851:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763978.301852:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763978.301853:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763978.301855:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763978/real 1787763978]  req@ffff9751f876da40 x1874599832678912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763978.301860:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763978.301861:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763978.301862:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763978.301863:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763978.301865:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763978.301866:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763978.301868:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876da40 x1874599832678912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763978.301873:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876da40 x1874599832678912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.301877:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763978.301878:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763978.301879:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763978.301880:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763978.301882:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876da40 x1874599832678912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.301887:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763978.301889:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876da40 x1874599832678912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.301893:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763978.301895:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763978.301895:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763978.301896:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763978.301897:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763978.301906:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763978.301908:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763978.301910:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763978.301911:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763978.301914:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876da40 x1874599832678912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763978.301918:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832678912:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763978.301921:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763978.301923:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cb40 x1874599832679040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.301928:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763978.301929:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763978.301930:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763978.301931:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763978.301932:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763978.301934:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832679040:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763978.301936:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763978.301938:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763978.301939:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763978.301940:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763978.301943:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763978.301944:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763978.301946:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763978.301947:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763978.301949:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763978.301951:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832679040, portal 10
00000100:00000001:2.0:1787763978.301952:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763978.301953:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049520960 : -115096660030656 : ffff9751f876cb40)
00000100:00000040:2.0:1787763978.301957:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cb40 x1874599832679040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764003 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.301960:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763978.301962:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763978.301963:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763978.301965:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832679040, offset 0
00000400:00000200:2.0:1787763978.301967:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763978.301969:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763978.301971:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763978.301973:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763978.301974:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763978.301976:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763978.301977:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763978.301979:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58540:58540:256:4294967295] 192.168.203.110@tcp LPNI seq info [58540:58540:8:4294967295]
00000400:00000200:2.0:1787763978.301982:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763978.301986:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763978.301989:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763978.301990:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763978.301993:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763978.301995:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787763978.301998:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763978.302000:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:2.0:1787763978.302002:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763978.302003:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763978.302005:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763978.302007:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763978.302009:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763978.302010:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763978.302012:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763978.302014:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cb40 x1874599832679040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764003 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.302019:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763978.302021:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cb40 x1874599832679040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764003 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.302025:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763978.302026:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763978.302028:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763978.302029:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787763978.302031:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763978.302032:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316365332
00000100:00000001:2.0:1787763978.302035:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316366580
00000100:00000001:2.0:1787763978.302036:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316365332
00000100:00000001:2.0:1787763978.302037:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316366580
00000100:00000001:2.0:1787763978.302039:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763978.302040:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763978.302041:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763978.302043:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763978/real 1787763978]  req@ffff9751f876cb40 x1874599832679040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763978.302056:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763978.302058:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763978.302059:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763978.302060:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763978.302062:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763978.302063:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763978.302065:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cb40 x1874599832679040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763978.302071:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cb40 x1874599832679040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.302075:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763978.302076:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763978.302077:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763978.302077:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763978.302080:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cb40 x1874599832679040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.302083:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763978.302086:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cb40 x1874599832679040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.302093:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763978.302095:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763978.302096:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763978.302097:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763978.302098:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763978.302099:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763978.302101:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763978.302103:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763978.302104:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763978.302107:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cb40 x1874599832679040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763978.302116:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832679040:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763978.302119:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763978.302121:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832679168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.302125:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763978.302126:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763978.302127:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763978.302128:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763978.302130:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763978.302131:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832679168:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763978.302149:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763978.302152:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763978.302154:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763978.302156:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763978.302159:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763978.302162:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763978.302165:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763978.302168:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763978.302172:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763978.302175:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832679168, portal 4
00000100:00000001:2.0:1787763978.302177:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763978.302179:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
00000100:00000040:2.0:1787763978.302185:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832679168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764003 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.302193:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763978.302195:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763978.302198:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763978.302200:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832679168, offset 0
00000400:00000200:2.0:1787763978.302204:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763978.302208:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763978.302210:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763978.302212:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763978.302214:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763978.302216:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763978.302218:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763978.302220:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58541:58541:256:4294967295] 192.168.203.110@tcp LPNI seq info [58541:58541:8:4294967295]
00000400:00000200:2.0:1787763978.302222:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763978.302226:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763978.302229:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763978.302231:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763978.302234:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763978.302236:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:2.1:1787763978.302238:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763978.302240:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:2.0:1787763978.302243:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763978.302244:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763978.302246:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763978.302248:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763978.302250:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763978.302251:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763978.302253:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763978.302255:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876c000 x1874599832679168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764003 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.302260:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763978.302262:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832679168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764003 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.302267:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763978.302268:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763978.302270:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763978.302272:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787763978.302274:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763978.302275:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316365350
00000100:00000001:2.0:1787763978.302277:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316366580
00000100:00000001:2.0:1787763978.302279:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316365350
00000100:00000001:2.0:1787763978.302280:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316366580
00000100:00000001:2.0:1787763978.302281:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763978.302282:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763978.302284:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763978.302286:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763978/real 1787763978]  req@ffff9751f876c000 x1874599832679168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763978.302290:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763978.302291:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763978.302293:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763978.302294:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763978.302295:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763978.302296:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763978.302298:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832679168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763978.302304:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832679168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.302307:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763978.302308:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763978.302309:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763978.302310:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763978.302312:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832679168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.302316:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763978.302319:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832679168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.302322:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763978.302325:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763978.302326:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763978.302326:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763978.302328:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763978.302329:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763978.302330:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763978.302332:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763978.302334:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763978.302336:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832679168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763978.302340:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832679168:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763978.302343:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763978.302344:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763978.302346:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876da40 x1874599832678912/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.302349:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763978.302350:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763978.302352:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763978.302353:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763978.302354:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763978.302355:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763978.302357:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763978.302358:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014800.
02000000:00000001:2.0:1787763978.302360:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763978.302361:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763978.302362:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876da40.
00000100:00000001:2.0:1787763978.302364:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763978.302365:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763978.302366:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763978.302368:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cb40 x1874599832679040/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.302390:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763978.302391:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763978.302393:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763978.302395:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763978.302396:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763978.302397:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763978.302398:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763978.302399:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:2.0:1787763978.302401:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763978.302401:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763978.302402:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cb40.
00000100:00000001:2.0:1787763978.302404:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763978.302405:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763978.302406:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763978.302409:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832679168/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764003 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763978.302413:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763978.302414:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763978.302415:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787763978.302417:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763978.302417:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763978.302418:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787763978.302419:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763978.302420:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015c00.
02000000:00000001:2.0:1787763978.302421:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763978.302422:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763978.302422:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:2.0:1787763978.302424:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763978.302424:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763978.302425:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763978.302428:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763978.302428:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763978.302430:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763978.302431:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763978.940286:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763978.940290:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763978.940296:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763978.940297:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763978.940302:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763978.940305:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787763978.940310:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763978.940310:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763978.940316:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763978.940317:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763978.940318:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763978.940323:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763978.940328:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763978.940331:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763978.940336:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763978.940383:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763978.940389:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763978.940393:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763979.324253:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763979.324264:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763979.324267:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763979.324275:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763979.324279:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763979.324284:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763979.324288:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763979.324292:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763979.324297:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763979.580260:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316366580, at 4316366592
00000100:00000010:3.0:1787763979.580279:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7cc0.
00000100:00000040:2.0:1787763979.580295:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316366580, at 4316366592
00000100:00000010:2.0:1787763979.580305:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00000100:00000001:1.0:1787763979.964269:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763979.964278:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763979.964280:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763979.964285:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763979.964285:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763979.964293:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763979.964294:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763979.964299:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763979.964304:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763979.964309:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763979.964309:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763979.964315:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763979.964316:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763979.964323:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763979.964325:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763979.964400:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763979.964404:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763979.964408:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763980.348214:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763980.348223:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763980.348226:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763980.348233:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763980.348237:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763980.348241:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763980.348243:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763980.348246:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763980.348248:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763980.988264:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763980.988264:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763980.988270:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763980.988272:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763980.988276:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763980.988276:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763980.988279:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763980.988282:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763980.988285:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763980.988288:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763980.988289:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763980.988296:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763980.988301:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763980.988305:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763980.988309:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763980.988346:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763980.988350:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763980.988353:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763981.372248:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763981.372258:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763981.372260:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763981.372266:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763981.372269:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763981.372273:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763981.372276:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763981.372279:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763981.372283:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763982.012227:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763982.012235:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763982.012242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763982.012243:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763982.012247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763982.012250:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763982.012253:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763982.012256:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763982.012259:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763982.012261:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763982.012263:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763982.012265:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763982.012266:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763982.012277:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763982.012280:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763982.012284:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763982.012289:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763982.012292:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763982.012295:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763982.012298:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763982.012300:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763982.012325:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763982.012328:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763982.012331:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763982.396258:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763982.396266:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763982.396268:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763982.396273:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763982.396277:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763982.396281:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763982.396283:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763982.396286:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763982.396290:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763983.036238:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763983.036239:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763983.036245:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763983.036246:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763983.036250:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763983.036252:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763983.036255:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763983.036256:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763983.036265:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763983.036266:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763983.036269:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763983.036272:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763983.036275:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763983.036277:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763983.036279:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787763983.036316:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787763983.036322:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787763983.036325:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763983.420276:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763983.420280:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:0.0:1787763983.420287:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787763983.420289:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787763983.420290:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787763983.420293:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763983.420295:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1787763983.420300:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787763983.420301:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763983.420307:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1787763983.420310:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21698
00000100:00000001:2.0:1787763983.420312:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1787763983.420321:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763983.420321:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763983.420328:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1787763983.420334:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787763983.420335:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763983.420339:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763983.420360:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763983.420365:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763983.420371:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763983.420375:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763983.420379:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763983.420382:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763983.420385:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763983.420388:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763983.420391:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763983.420393:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763983.420395:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763983.420397:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763983.420399:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763983.420401:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763983.420404:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763983.420407:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763983.420409:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763983.420411:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763983.420414:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763983.420417:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763983.420419:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763983.420421:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763983.420430:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787763983.420433:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763983.420439:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763983.420441:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763983.420445:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763983.420448:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763983.420452:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763983.420461:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787763983.420468:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763983.420478:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832679296/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763983.420488:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763983.420493:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763983.420500:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832679296/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763983.420522:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763983.420527:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763983.420532:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763983.420534:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763983.420536:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763983.420540:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763983.420545:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21698
00000400:00000200:0.0:1787763983.420551:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763983.420558:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787763983.420559:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:0.0:1787763983.420561:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763983.420566:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763983.420566:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763983.420571:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763983.420572:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787763983.420577:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763983.420579:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763983.420582:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763983.420583:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763983.420585:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:2.0:1787763983.420585:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832679296/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787763983.420588:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763983.420592:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787763983.420597:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763983.420604:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763983.420606:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763983.420607:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763983.420610:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763983.420611:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763983.420617:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763983.420620:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:2.0:1787763983.420620:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:0.0:1787763983.420626:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:2.0:1787763983.420627:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763983.420628:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763983.420634:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763983.420634:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763983.420642:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:2.0:1787763983.420644:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832679296:192.168.203.110@tcp:250:kworker.0
02000000:00000001:0.0:1787763983.420646:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763983.420650:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787763983.420656:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:0.0:1787763983.420657:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763983.420661:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763983.420662:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763983.420663:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:0.0:1787763983.420665:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
02000000:00000001:2.0:1787763983.420668:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787763983.420669:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763983.420675:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763983.420675:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1787763983.420679:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763983.420681:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763983.420682:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763983.420683:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763983.420686:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000010:0.0:1787763983.420690:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
02000000:00000001:2.0:1787763983.420691:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763983.420696:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763983.420702:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832679424/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787763983.420703:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763983.420712:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:2.0:1787763983.420713:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000100:00000001:0.0:1787763983.420714:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763983.420720:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832679424/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1787763983.420725:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000001:0.0:1787763983.420727:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763983.420732:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763983.420735:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763983.420736:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000200:2.0:1787763983.420737:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832679296, portal 25
00000100:00080000:0.0:1787763983.420738:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763983.420742:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763983.420744:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00080000:0.0:1787763983.420745:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21698
00000100:00000001:2.0:1787763983.420750:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000400:00000200:0.0:1787763983.420752:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763983.420758:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763983.420760:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763983.420765:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000040:2.0:1787763983.420765:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832679296/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764008 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763983.420769:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763983.420773:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763983.420776:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763983.420782:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763983.420784:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000100:00000001:2.0:1787763983.420785:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000001:0.0:1787763983.420786:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763983.420790:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763983.420792:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:2.0:1787763983.420796:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000020:00000001:0.0:1787763983.420797:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787763983.420803:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000010:2.0:1787763983.420804:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000001:0.0:1787763983.420806:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763983.420813:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000200:2.0:1787763983.420814:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832679296, offset 0
00000100:00000001:0.0:1787763983.420815:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763983.420822:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763983.420830:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:2.0:1787763983.420835:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763983.420836:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763983.420841:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763983.420845:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763983.420847:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763983.420850:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763983.420852:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763983.420855:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763983.420859:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763983.420860:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763983.420867:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000400:00000200:2.0:1787763983.420872:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000020:00000040:0.0:1787763983.420873:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763983.420883:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:2.0:1787763983.420884:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:0.0:1787763983.420886:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763983.420891:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763983.420894:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1787763983.420895:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
02000000:00000001:0.0:1787763983.420899:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763983.420904:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
02000000:00000010:0.0:1787763983.420907:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000400:00000200:2.0:1787763983.420910:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:0.0:1787763983.420914:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763983.420922:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58542:58542:256:4294967295] 192.168.203.110@tcp LPNI seq info [58542:58542:8:4294967295]
00000100:00100000:0.0:1787763983.420924:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832679552/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787763983.420934:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763983.420943:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763983.420945:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763983.420951:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832679552/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787763983.420958:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1787763983.420964:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763983.420968:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763983.420975:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:2.0:1787763983.420975:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00080000:0.0:1787763983.420979:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763983.420982:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763983.420983:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763983.420984:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763983.420986:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000800:00000010:2.0:1787763983.420986:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000100:00080000:0.0:1787763983.420991:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21698
00000400:00000200:0.0:1787763983.420995:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763983.420999:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000800:00000200:2.0:1787763983.421001:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00080000:0.0:1787763983.421002:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763983.421009:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763983.421011:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1787763983.421011:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000100:00000040:0.0:1787763983.421016:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763983.421020:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763983.421021:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763983.421023:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787763983.421026:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:2.1:1787763983.421026:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:0.0:1787763983.421028:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763983.421031:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787763983.421034:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000800:00000200:2.1:1787763983.421035:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000100:00000040:0.0:1787763983.421038:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763983.421042:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763983.421052:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763983.421056:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000100:2.0:1787763983.421057:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000020:00000040:0.0:1787763983.421059:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787763983.421063:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000800:00000010:2.0:1787763983.421063:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000100:00000001:0.0:1787763983.421069:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763983.421071:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763983.421073:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763983.421075:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763983.421075:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1787763983.421080:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763983.421084:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000400:00000200:2.0:1787763983.421087:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000008:00000001:0.0:1787763983.421088:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763983.421092:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763983.421096:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787763983.421098:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000400:00000200:2.0:1787763983.421099:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
02000000:00000001:0.0:1787763983.421101:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763983.421104:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763983.421105:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:2.0:1787763983.421105:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
02000000:00000001:0.0:1787763983.421108:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763983.421111:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763983.421115:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:2.0:1787763983.421115:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1787763983.421120:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763983.421124:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e1c0 x1874599832679296/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764008 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787763983.421127:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832679680/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763983.421162:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763983.421165:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763983.421169:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832679680/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763983.421190:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763983.421194:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787763983.421196:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21708)
00000100:00000040:2.0:1787763983.421203:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832679296/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764008 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.421215:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763983.421217:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763983.421226:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763983.421231:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763983.421234:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316371700
00000100:00000010:2.0:1787763983.421241:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00000001:2.0:1787763983.421249:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787763983.421257:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316371700
00000100:00000010:2.0:1787763983.421261:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7660.
00000100:00000001:2.0:1787763983.421265:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787763983.421269:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763983.421272:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763983.421275:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763983.421281:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763983/real 1787763983]  req@ffff9751f876e1c0 x1874599832679296/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763983.421292:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763983.421294:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763983.421297:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763983.421300:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763983.421302:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763983.421306:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763983.421310:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832679296/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763983.421321:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832679296/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.421329:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763983.421331:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763983.421335:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763983.421337:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763983.421359:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832679296/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.421368:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763983.421374:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832679296/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.421382:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763983.421385:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763983.421387:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763983.421388:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763983.421392:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763983.421394:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763983.421398:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763983.421402:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763983.421405:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763983.421410:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832679296/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763983.421421:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832679296:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787763983.421426:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763983.421428:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763983.421432:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832679296/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.421439:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763983.421442:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763983.421446:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763983.421450:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763983.421452:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763983.421454:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787763983.421457:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763983.421460:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016800.
02000000:00000001:2.0:1787763983.421466:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763983.421469:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763983.421471:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:2.0:1787763983.421476:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763983.421478:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763983.421480:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763983.421489:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763983.421491:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763983.421494:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763983.421496:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763983.421497:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763983.421502:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832679424/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.421511:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763983.421513:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763983.421516:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763983.421518:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763983.421521:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763983.421525:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832679424:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763983.421529:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763983.421531:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763983.421532:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763983.421535:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763983.421538:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763983.421540:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763983.421544:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763983.421548:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763983.421552:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763983.421556:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832679424, portal 10
00000100:00000001:2.0:1787763983.421558:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763983.421559:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00000040:2.0:1787763983.421567:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832679424/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764008 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.421574:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763983.421577:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763983.421580:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763983.421582:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832679424, offset 0
00000400:00000200:2.0:1787763983.421587:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763983.421591:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763983.421595:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763983.421599:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763983.421602:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763983.421604:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763983.421607:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763983.421611:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58543:58543:256:4294967295] 192.168.203.110@tcp LPNI seq info [58543:58543:8:4294967295]
00000400:00000200:2.0:1787763983.421615:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763983.421623:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763983.421627:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763983.421631:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763983.421635:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763983.421638:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787763983.421643:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763983.421646:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:2.0:1787763983.421650:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763983.421652:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763983.421656:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763983.421660:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763983.421663:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763983.421666:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763983.421669:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763983.421673:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e580 x1874599832679424/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764008 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.421683:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763983.421687:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832679424/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764008 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.421695:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763983.421697:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763983.421701:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763983.421706:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787763983.421710:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763983.421713:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316370464
00000100:00000001:2.0:1787763983.421716:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316371700
00000100:00000001:2.0:1787763983.421719:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316370464
00000100:00000001:2.0:1787763983.421722:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316371700
00000100:00000001:2.0:1787763983.421724:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763983.421727:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763983.421729:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763983.421733:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763983/real 1787763983]  req@ffff9751f876e580 x1874599832679424/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763983.421741:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763983.421743:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763983.421745:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763983.421748:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763983.421750:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763983.421752:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763983.421755:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832679424/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763983.421765:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832679424/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.421772:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763983.421774:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763983.421776:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763983.421777:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763983.421781:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832679424/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.421788:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763983.421793:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832679424/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.421802:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763983.421805:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763983.421806:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763983.421808:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763983.421810:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763983.421822:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763983.421825:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763983.421828:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763983.421831:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763983.421835:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832679424/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763983.421844:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832679424:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763983.421849:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763983.421852:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f0c0 x1874599832679552/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.421860:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763983.421861:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763983.421863:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763983.421865:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763983.421868:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763983.421871:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832679552:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763983.421876:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763983.421878:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763983.421880:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763983.421882:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763983.421886:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763983.421888:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763983.421892:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763983.421895:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763983.421899:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763983.421902:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832679552, portal 10
00000100:00000001:2.0:1787763983.421905:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763983.421906:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049530560 : -115096660021056 : ffff9751f876f0c0)
00000100:00000040:2.0:1787763983.421913:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f0c0 x1874599832679552/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764008 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.421921:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763983.421924:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763983.421927:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763983.421929:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832679552, offset 0
00000400:00000200:2.0:1787763983.421934:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763983.421938:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763983.421942:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763983.421945:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763983.421948:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763983.421951:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763983.421954:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763983.421957:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58544:58544:256:4294967295] 192.168.203.110@tcp LPNI seq info [58544:58544:8:4294967295]
00000400:00000200:2.0:1787763983.421962:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763983.421968:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763983.421973:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763983.421977:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763983.421981:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763983.421984:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787763983.421989:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763983.421992:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:2.0:1787763983.421996:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763983.421999:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763983.422002:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763983.422005:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763983.422008:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763983.422011:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763983.422014:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763983.422018:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f0c0 x1874599832679552/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764008 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.422028:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763983.422031:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f0c0 x1874599832679552/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764008 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.422039:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763983.422041:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763983.422044:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763983.422047:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787763983.422051:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763983.422053:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316370452
00000100:00000001:2.0:1787763983.422056:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316371700
00000100:00000001:2.0:1787763983.422058:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316370452
00000100:00000001:2.0:1787763983.422061:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316371700
00000100:00000001:2.0:1787763983.422064:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763983.422066:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763983.422069:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763983.422073:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763983/real 1787763983]  req@ffff9751f876f0c0 x1874599832679552/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763983.422082:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763983.422084:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763983.422086:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763983.422089:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763983.422091:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763983.422093:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763983.422097:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f0c0 x1874599832679552/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763983.422108:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f0c0 x1874599832679552/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.422116:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763983.422118:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763983.422120:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763983.422121:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763983.422126:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f0c0 x1874599832679552/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.422166:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763983.422173:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f0c0 x1874599832679552/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.422181:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763983.422184:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763983.422186:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763983.422187:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763983.422189:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763983.422191:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763983.422193:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763983.422197:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763983.422199:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763983.422204:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f0c0 x1874599832679552/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763983.422212:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832679552:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763983.422216:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763983.422229:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832679680/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.422240:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763983.422242:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763983.422245:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763983.422248:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763983.422250:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763983.422253:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832679680:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763983.422257:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763983.422259:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763983.422261:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763983.422263:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763983.422267:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763983.422270:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763983.422274:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763983.422277:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763983.422281:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763983.422284:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832679680, portal 4
00000100:00000001:2.0:1787763983.422286:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763983.422288:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
00000100:00000040:2.0:1787763983.422295:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832679680/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764008 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.422303:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763983.422306:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763983.422309:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763983.422311:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832679680, offset 0
00000400:00000200:2.0:1787763983.422316:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763983.422321:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763983.422325:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763983.422328:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763983.422331:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763983.422334:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763983.422336:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763983.422339:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58545:58545:256:4294967295] 192.168.203.110@tcp LPNI seq info [58545:58545:8:4294967295]
00000400:00000200:2.0:1787763983.422360:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763983.422367:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763983.422373:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763983.422376:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763983.422381:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763983.422383:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:2.1:1787763983.422388:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763983.422391:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:2.0:1787763983.422396:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763983.422398:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763983.422401:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763983.422404:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763983.422408:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763983.422410:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763983.422412:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763983.422416:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876ed00 x1874599832679680/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764008 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.422424:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763983.422428:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832679680/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764008 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.422435:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763983.422437:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763983.422440:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763983.422444:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787763983.422448:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763983.422450:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316370470
00000100:00000001:2.0:1787763983.422453:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316371700
00000100:00000001:2.0:1787763983.422455:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316370470
00000100:00000001:2.0:1787763983.422457:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316371700
00000100:00000001:2.0:1787763983.422459:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763983.422462:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763983.422464:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763983.422468:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763983/real 1787763983]  req@ffff9751f876ed00 x1874599832679680/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763983.422479:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763983.422481:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763983.422483:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763983.422486:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763983.422488:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763983.422491:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763983.422495:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832679680/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763983.422505:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832679680/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.422513:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763983.422514:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763983.422517:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763983.422518:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763983.422523:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832679680/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.422531:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763983.422536:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832679680/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.422544:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763983.422546:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763983.422548:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763983.422550:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763983.422552:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763983.422554:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763983.422557:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763983.422560:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763983.422563:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763983.422568:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832679680/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763983.422578:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832679680:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763983.422582:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763983.422584:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763983.422588:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832679424/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.422595:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763983.422596:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763983.422599:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763983.422602:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763983.422604:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763983.422605:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763983.422607:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763983.422609:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015c00.
02000000:00000001:2.0:1787763983.422612:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763983.422614:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763983.422615:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:2.0:1787763983.422618:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763983.422620:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763983.422621:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763983.422625:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f0c0 x1874599832679552/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.422632:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763983.422633:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763983.422635:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763983.422637:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763983.422639:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763983.422640:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763983.422642:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763983.422644:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:2.0:1787763983.422646:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763983.422647:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763983.422649:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f0c0.
00000100:00000001:2.0:1787763983.422651:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763983.422652:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763983.422654:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763983.422658:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832679680/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764008 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763983.422666:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763983.422667:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763983.422669:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763983.422672:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763983.422673:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763983.422675:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787763983.422677:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763983.422679:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014800.
02000000:00000001:2.0:1787763983.422682:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763983.422683:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763983.422685:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:2.0:1787763983.422687:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763983.422689:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763983.422691:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763983.422695:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763983.422697:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763983.422699:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763983.422701:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763984.060230:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763984.060240:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787763984.060243:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763984.060246:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763984.060250:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:1.0:1787763984.060252:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763984.060252:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763984.060255:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763984.060259:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763984.060261:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763984.060262:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763984.060266:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763984.060267:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763984.060269:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763984.060272:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763984.060274:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763984.060280:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763984.060283:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763984.060286:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763984.060288:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763984.060290:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763984.060333:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763984.060350:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763984.060354:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763984.444264:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763984.444276:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763984.444279:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763984.444287:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763984.444291:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763984.444296:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763984.444299:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763984.444302:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763984.444306:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763984.700296:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316371700, at 4316371712
00000100:00000040:2.0:1787763984.700313:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316371700, at 4316371712
00000100:00000010:3.0:1787763984.700313:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7660.
00000100:00000010:2.0:1787763984.700331:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00010000:00000001:3.0:1787763985.084204:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763985.084205:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763985.084210:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763985.084212:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763985.084216:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763985.084217:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763985.084220:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763985.084223:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763985.084224:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763985.084226:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763985.084228:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763985.084233:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763985.084236:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763985.084238:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763985.084241:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763985.084273:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763985.084278:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763985.084282:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763985.468254:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763985.468265:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763985.468268:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763985.468276:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763985.468280:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763985.468284:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763985.468286:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763985.468289:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763985.468292:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763986.108244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763986.108252:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763986.108259:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763986.108260:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763986.108262:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763986.108267:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763986.108268:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763986.108274:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763986.108278:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763986.108283:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763986.108288:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763986.108291:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763986.108295:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763986.108298:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763986.108302:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763986.108313:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763986.108335:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763986.108338:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763986.492254:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763986.492261:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763986.492263:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763986.492269:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763986.492272:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763986.492276:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763986.492278:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763986.492280:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763986.492283:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763987.132261:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763987.132269:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763987.132270:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763987.132280:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763987.132281:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763987.132283:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763987.132287:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00100000:1.0:1787763987.132289:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763987.132291:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:1.0:1787763987.132297:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763987.132299:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763987.132302:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763987.132305:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:1.0:1787763987.132307:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763987.132309:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763987.132311:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763987.132314:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763987.132331:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763987.132332:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763987.132337:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763987.132341:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763987.132369:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763987.132373:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763987.132375:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763987.516268:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763987.516276:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763987.516279:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763987.516285:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763987.516290:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763987.516294:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763987.516297:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763987.516300:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763987.516302:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763988.156230:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763988.156233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763988.156238:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763988.156239:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763988.156243:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763988.156247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787763988.156249:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763988.156251:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763988.156255:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763988.156259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763988.156260:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763988.156262:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763988.156264:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763988.156267:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763988.156270:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763988.156333:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763988.156341:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763988.156345:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763988.540239:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763988.540258:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763988.540261:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763988.540268:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763988.540268:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787763988.540280:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787763988.540280:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763988.540285:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763988.540289:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21703
00000100:00100000:2.0:1787763988.540295:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000400:00000200:0.0:1787763988.540304:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763988.540323:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000400:00000200:0.0:1787763988.540328:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787763988.540330:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787763988.540332:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763988.540339:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763988.540339:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787763988.540348:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763988.540349:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763988.540352:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763988.540359:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763988.540361:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763988.540364:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763988.540368:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763988.540370:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763988.540373:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763988.540376:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763988.540379:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763988.540381:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763988.540383:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763988.540385:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763988.540386:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763988.540388:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763988.540395:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763988.540397:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763988.540399:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763988.540403:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763988.540405:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763988.540406:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763988.540408:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763988.540416:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000020:00000040:0.0:1787763988.540419:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763988.540422:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763988.540423:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763988.540425:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763988.540427:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763988.540430:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763988.540437:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787763988.540442:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763988.540450:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832679808/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763988.540457:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763988.540461:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763988.540465:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832679808/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763988.540482:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763988.540485:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763988.540488:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763988.540489:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763988.540490:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763988.540493:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763988.540495:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21703
00000400:00000200:0.0:1787763988.540497:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763988.540499:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763988.540501:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763988.540503:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763988.540506:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763988.540508:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763988.540510:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763988.540512:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1787763988.540519:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:0.0:1787763988.540520:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763988.540524:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763988.540530:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000001:2.0:1787763988.540530:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1787763988.540536:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787763988.540536:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1787763988.540548:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787763988.540552:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763988.540553:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763988.540559:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763988.540563:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:2.0:1787763988.540564:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cf00 x1874599832679808/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787763988.540565:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787763988.540573:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763988.540580:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763988.540588:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763988.540595:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763988.540596:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:0.0:1787763988.540605:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763988.540605:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763988.540622:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1787763988.540624:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:0.0:1787763988.540628:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1787763988.540628:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763988.540637:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1787763988.540638:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763988.540653:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763988.540655:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832679808:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787763988.540661:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000010:0.0:1787763988.540663:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
02000000:00000001:2.0:1787763988.540667:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:0.0:1787763988.540669:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787763988.540671:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763988.540675:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763988.540676:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1787763988.540681:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763988.540684:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763988.540687:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:2.0:1787763988.540687:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763988.540691:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763988.540692:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763988.540699:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000010:0.0:1787763988.540700:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787763988.540706:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763988.540706:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000100:00100000:0.0:1787763988.540726:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832679936/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:2.0:1787763988.540727:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000001:0.0:1787763988.540740:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:2.0:1787763988.540741:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832679808, portal 25
00000100:00000001:2.0:1787763988.540749:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787763988.540750:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763988.540755:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049521920 : -115096660029696 : ffff9751f876cf00)
00000100:00000040:0.0:1787763988.540761:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832679936/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787763988.540768:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cf00 x1874599832679808/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764013 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763988.540796:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763988.540807:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787763988.540808:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:0.0:1787763988.540814:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763988.540822:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:2.0:1787763988.540826:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00080000:0.0:1787763988.540827:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763988.540836:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:2.0:1787763988.540838:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00080000:0.0:1787763988.540840:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21703
00000400:00000200:0.0:1787763988.540850:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000200:2.0:1787763988.540850:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832679808, offset 0
00000400:00000200:0.0:1787763988.540861:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763988.540864:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763988.540868:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763988.540869:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763988.540878:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787763988.540879:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787763988.540883:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763988.540887:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763988.540888:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763988.540889:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763988.540892:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763988.540894:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763988.540903:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:2.0:1787763988.540903:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:0.0:1787763988.540912:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763988.540914:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763988.540917:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787763988.540918:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:0.0:1787763988.540920:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763988.540925:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:2.0:1787763988.540927:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763988.540934:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00080000:0.0:1787763988.540939:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:2.0:1787763988.540945:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1787763988.540946:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763988.540953:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1787763988.540953:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
02000000:00000001:0.0:1787763988.540961:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1787763988.540966:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58546:58546:256:4294967295] 192.168.203.110@tcp LPNI seq info [58546:58546:8:4294967295]
02000000:00000001:0.0:1787763988.540967:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763988.540973:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763988.540975:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763988.540977:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763988.540978:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763988.540989:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000400:00000200:2.0:1787763988.540990:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:0.0:1787763988.540998:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763988.541003:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763988.541007:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763988.541013:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763988.541020:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763988.541024:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763988.541024:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
02000000:00000010:0.0:1787763988.541030:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787763988.541036:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763988.541043:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832680064/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.0:1787763988.541045:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763988.541063:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000100:00000001:0.0:1787763988.541067:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763988.541073:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763988.541077:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832680064/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1787763988.541087:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763988.541099:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787763988.541101:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000100:00000040:0.0:1787763988.541103:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763988.541109:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787763988.541118:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:2.1:1787763988.541119:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763988.541158:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1787763988.541165:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763988.541168:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000100:00080000:0.0:1787763988.541169:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763988.541174:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763988.541176:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763988.541178:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:2.0:1787763988.541180:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:0.0:1787763988.541183:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21703
00000400:00000200:0.0:1787763988.541190:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763988.541193:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787763988.541198:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1787763988.541200:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000100:00080000:0.0:1787763988.541201:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000010:2.0:1787763988.541206:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000040:0.0:1787763988.541209:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763988.541212:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763988.541213:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:0.0:1787763988.541216:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763988.541222:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000200:2.0:1787763988.541223:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cf00 x1874599832679808/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764013 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787763988.541233:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763988.541239:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000100:00000001:2.0:1787763988.541242:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000001:0.0:1787763988.541245:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:2.0:1787763988.541247:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cf00 x1874599832679808/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764013 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787763988.541250:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763988.541262:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:2.0:1787763988.541263:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763988.541271:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:0.0:1787763988.541273:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:2.0:1787763988.541279:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763988.541286:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:2.0:1787763988.541293:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763988.541294:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763988.541300:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316376820
00000100:00000040:0.0:1787763988.541302:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763988.541322:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763988.541326:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000010:2.0:1787763988.541328:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00080000:0.0:1787763988.541332:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763988.541335:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763988.541337:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787763988.541338:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
02000000:00000001:0.0:1787763988.541343:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763988.541347:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763988.541349:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787763988.541350:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316376820
00000008:00100000:0.0:1787763988.541352:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000010:2.0:1787763988.541355:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb73c0.
00000008:00000001:0.0:1787763988.541356:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763988.541363:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763988.541364:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000010:0.0:1787763988.541371:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000100:00000001:2.0:1787763988.541371:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763988.541382:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787763988.541383:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:2.0:1787763988.541386:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000001:0.0:1787763988.541388:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763988.541390:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763988.541392:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00100000:2.0:1787763988.541396:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763988/real 1787763988]  req@ffff9751f876cf00 x1874599832679808/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787763988.541397:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763988.541402:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763988.541416:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000400:00000001:2.0:1787763988.541417:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763988.541422:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:0.0:1787763988.541423:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763988.541425:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763988.541429:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000100:00100000:0.0:1787763988.541434:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832680192/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:2.0:1787763988.541434:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763988.541444:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763988.541453:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cf00 x1874599832679808/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763988.541457:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763988.541464:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763988.541478:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832680192/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763988.541485:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cf00 x1874599832679808/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763988.541507:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763988.541520:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000040:0.0:1787763988.541521:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21713)
00000100:00000001:2.0:1787763988.541524:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763988.541528:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763988.541532:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763988.541536:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cf00 x1874599832679808/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.541541:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763988.541547:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cf00 x1874599832679808/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.541552:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763988.541554:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763988.541556:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763988.541558:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763988.541561:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763988.541564:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763988.541568:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763988.541572:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763988.541575:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763988.541578:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cf00 x1874599832679808/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763988.541584:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832679808:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787763988.541588:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763988.541589:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763988.541591:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cf00 x1874599832679808/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.541595:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763988.541597:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763988.541600:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763988.541603:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763988.541606:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763988.541608:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787763988.541610:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763988.541613:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014400.
02000000:00000001:2.0:1787763988.541616:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763988.541618:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763988.541620:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cf00.
00000100:00000001:2.0:1787763988.541624:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763988.541627:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763988.541629:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763988.541636:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763988.541638:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763988.541640:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763988.541642:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763988.541643:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763988.541646:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f840 x1874599832679936/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.541650:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763988.541652:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763988.541653:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763988.541655:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763988.541658:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763988.541660:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832679936:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763988.541662:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763988.541664:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763988.541665:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763988.541667:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763988.541669:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763988.541670:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763988.541674:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763988.541676:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763988.541679:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763988.541681:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832679936, portal 10
00000100:00000001:2.0:1787763988.541682:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763988.541684:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049532480 : -115096660019136 : ffff9751f876f840)
00000100:00000040:2.0:1787763988.541689:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f840 x1874599832679936/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764013 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.541694:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763988.541695:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763988.541697:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763988.541698:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832679936, offset 0
00000400:00000200:2.0:1787763988.541701:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763988.541704:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763988.541707:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763988.541709:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763988.541711:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763988.541713:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763988.541714:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763988.541717:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58547:58547:256:4294967295] 192.168.203.110@tcp LPNI seq info [58547:58547:8:4294967295]
00000400:00000200:2.0:1787763988.541720:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763988.541725:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763988.541728:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763988.541731:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763988.541734:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763988.541736:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787763988.541740:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763988.541742:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1787763988.541745:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763988.541746:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763988.541748:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763988.541750:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763988.541753:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763988.541754:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763988.541756:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763988.541758:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f840 x1874599832679936/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764013 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.541764:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763988.541766:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f840 x1874599832679936/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764013 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.541771:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763988.541772:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763988.541774:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763988.541777:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787763988.541778:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763988.541780:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316375584
00000100:00000001:2.0:1787763988.541782:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316376820
00000100:00000001:2.0:1787763988.541784:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316375584
00000100:00000001:2.0:1787763988.541785:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316376820
00000100:00000001:2.0:1787763988.541787:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763988.541788:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763988.541789:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763988.541791:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763988/real 1787763988]  req@ffff9751f876f840 x1874599832679936/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763988.541801:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763988.541802:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763988.541804:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763988.541805:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763988.541806:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763988.541807:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763988.541809:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f840 x1874599832679936/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763988.541814:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f840 x1874599832679936/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.541818:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763988.541819:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763988.541820:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763988.541821:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763988.541824:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f840 x1874599832679936/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.541828:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763988.541830:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f840 x1874599832679936/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.541834:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763988.541836:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763988.541837:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763988.541837:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763988.541838:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763988.541848:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763988.541850:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763988.541852:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763988.541853:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763988.541856:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f840 x1874599832679936/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763988.541862:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832679936:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763988.541864:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763988.541866:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e940 x1874599832680064/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.541871:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763988.541872:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763988.541873:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763988.541874:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763988.541875:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763988.541877:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832680064:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763988.541879:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763988.541880:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763988.541881:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763988.541882:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763988.541884:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763988.541886:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763988.541888:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763988.541889:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763988.541891:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763988.541893:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832680064, portal 10
00000100:00000001:2.0:1787763988.541894:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763988.541895:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049528640 : -115096660022976 : ffff9751f876e940)
00000100:00000040:2.0:1787763988.541899:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e940 x1874599832680064/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764013 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.541904:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763988.541905:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763988.541906:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763988.541908:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832680064, offset 0
00000400:00000200:2.0:1787763988.541911:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763988.541913:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763988.541915:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763988.541917:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763988.541919:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763988.541921:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763988.541922:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763988.541924:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58548:58548:256:4294967295] 192.168.203.110@tcp LPNI seq info [58548:58548:8:4294967295]
00000400:00000200:2.0:1787763988.541927:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763988.541931:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763988.541934:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763988.541935:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763988.541937:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763988.541940:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787763988.541942:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763988.541944:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1787763988.541947:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763988.541948:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763988.541950:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763988.541952:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763988.541954:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763988.541955:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763988.541956:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763988.541959:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e940 x1874599832680064/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764013 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.541964:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763988.541966:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e940 x1874599832680064/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764013 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.541970:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763988.541971:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763988.541972:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763988.541974:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787763988.541976:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763988.541978:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316375572
00000100:00000001:2.0:1787763988.541980:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316376820
00000100:00000001:2.0:1787763988.541982:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316375572
00000100:00000001:2.0:1787763988.541983:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316376820
00000100:00000001:2.0:1787763988.541984:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763988.541985:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763988.541987:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763988.541989:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763988/real 1787763988]  req@ffff9751f876e940 x1874599832680064/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763988.541993:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763988.541994:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763988.541995:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763988.541996:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763988.541997:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763988.541998:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763988.542000:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e940 x1874599832680064/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763988.542006:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e940 x1874599832680064/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.542011:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763988.542012:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763988.542013:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763988.542015:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763988.542018:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e940 x1874599832680064/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.542026:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763988.542030:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e940 x1874599832680064/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.542038:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763988.542041:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763988.542043:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763988.542044:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763988.542046:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763988.542048:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763988.542051:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763988.542054:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763988.542057:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763988.542061:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e940 x1874599832680064/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763988.542070:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832680064:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763988.542073:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763988.542077:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832680192/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.542084:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763988.542086:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763988.542088:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763988.542090:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763988.542092:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763988.542095:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832680192:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763988.542098:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763988.542100:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763988.542102:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763988.542104:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763988.542108:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763988.542109:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763988.542113:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763988.542116:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763988.542120:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763988.542123:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832680192, portal 4
00000100:00000001:2.0:1787763988.542125:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763988.542126:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
00000100:00000040:2.0:1787763988.542132:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832680192/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764013 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.542163:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763988.542165:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763988.542168:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763988.542170:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832680192, offset 0
00000400:00000200:2.0:1787763988.542174:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763988.542178:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763988.542182:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763988.542185:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763988.542187:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763988.542190:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763988.542192:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763988.542195:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58549:58549:256:4294967295] 192.168.203.110@tcp LPNI seq info [58549:58549:8:4294967295]
00000400:00000200:2.0:1787763988.542199:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763988.542205:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763988.542210:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763988.542213:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763988.542216:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763988.542219:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:2.1:1787763988.542223:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763988.542226:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:2.0:1787763988.542230:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763988.542232:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763988.542235:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763988.542239:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763988.542242:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763988.542244:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763988.542246:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763988.542250:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876ed00 x1874599832680192/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764013 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.542260:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763988.542262:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832680192/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764013 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.542267:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763988.542268:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763988.542271:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763988.542273:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787763988.542275:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763988.542277:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316375590
00000100:00000001:2.0:1787763988.542279:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316376820
00000100:00000001:2.0:1787763988.542280:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316375590
00000100:00000001:2.0:1787763988.542282:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316376820
00000100:00000001:2.0:1787763988.542283:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763988.542285:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763988.542286:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763988.542288:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763988/real 1787763988]  req@ffff9751f876ed00 x1874599832680192/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763988.542294:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763988.542295:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763988.542296:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763988.542298:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763988.542299:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763988.542300:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763988.542302:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832680192/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763988.542322:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832680192/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.542327:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763988.542329:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763988.542330:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763988.542331:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763988.542334:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832680192/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.542338:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763988.542340:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832680192/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.542344:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763988.542346:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763988.542347:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763988.542348:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763988.542350:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763988.542351:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763988.542353:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763988.542355:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763988.542356:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763988.542359:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832680192/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763988.542363:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832680192:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763988.542366:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763988.542367:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763988.542369:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f840 x1874599832679936/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.542373:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763988.542374:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763988.542375:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763988.542377:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763988.542378:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763988.542379:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763988.542381:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763988.542382:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017000.
02000000:00000001:2.0:1787763988.542385:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763988.542386:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763988.542387:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f840.
00000100:00000001:2.0:1787763988.542389:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763988.542391:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763988.542392:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763988.542394:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e940 x1874599832680064/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.542398:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763988.542398:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763988.542399:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763988.542401:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763988.542401:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763988.542402:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763988.542403:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763988.542404:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:2.0:1787763988.542405:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763988.542406:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763988.542407:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e940.
00000100:00000001:2.0:1787763988.542410:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763988.542411:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763988.542412:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763988.542414:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832680192/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764013 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763988.542418:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763988.542419:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763988.542420:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787763988.542421:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763988.542422:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763988.542423:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787763988.542424:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763988.542425:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
02000000:00000001:2.0:1787763988.542426:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763988.542428:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763988.542429:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:2.0:1787763988.542430:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763988.542431:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763988.542432:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763988.542435:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763988.542436:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763988.542437:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763988.542438:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763989.180272:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763989.180281:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763989.180284:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763989.180289:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763989.180289:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763989.180298:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763989.180300:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763989.180316:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763989.180322:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763989.180323:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763989.180333:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763989.180334:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763989.180345:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763989.180349:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763989.180353:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763989.180418:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763989.180421:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763989.180425:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763989.564246:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763989.564255:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763989.564257:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763989.564265:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763989.564269:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763989.564273:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763989.564276:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763989.564278:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763989.564281:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763989.820222:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316376820, at 4316376832
00000100:00000010:3.0:1787763989.820241:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb73c0.
00000100:00000040:2.0:1787763989.820249:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316376820, at 4316376832
00000100:00000010:2.0:1787763989.820258:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00010000:00000001:3.0:1787763990.204235:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763990.204242:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763990.204251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763990.204254:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763990.204256:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763990.204264:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763990.204266:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763990.204267:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763990.204270:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763990.204276:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763990.204283:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763990.204287:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763990.204291:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763990.204309:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763990.204315:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763990.204341:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763990.204345:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763990.204348:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763990.588270:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763990.588279:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763990.588281:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763990.588288:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763990.588306:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763990.588312:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763990.588315:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763990.588318:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763990.588322:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763991.228237:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763991.228241:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763991.228246:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763991.228247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763991.228250:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763991.228255:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763991.228257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763991.228260:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763991.228260:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763991.228264:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763991.228265:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763991.228266:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763991.228269:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763991.228272:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763991.228275:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763991.228342:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763991.228350:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763991.228355:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763991.612264:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763991.612273:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763991.612275:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763991.612280:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763991.612283:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763991.612305:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763991.612308:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763991.612311:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763991.612314:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763992.252217:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763992.252219:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763992.252225:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763992.252226:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763992.252230:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763992.252233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787763992.252235:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763992.252236:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763992.252241:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763992.252243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763992.252246:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763992.252248:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:1.0:1787763992.252250:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763992.252252:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:1.0:1787763992.252254:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763992.252257:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763992.252258:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763992.252261:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763992.252263:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763992.252265:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763992.252267:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763992.252320:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763992.252325:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763992.252328:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763992.636272:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763992.636299:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763992.636302:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763992.636309:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763992.636315:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763992.636319:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763992.636323:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763992.636326:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763992.636330:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763993.276217:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787763993.276224:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763993.276227:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763993.276227:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787763993.276232:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763993.276234:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763993.276236:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763993.276243:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763993.276245:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763993.276247:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763993.276250:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763993.276251:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763993.276258:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763993.276261:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763993.276265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763993.276339:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763993.276343:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763993.276345:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787763993.660243:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763993.660251:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787763993.660253:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763993.660256:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763993.660262:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763993.660268:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21708
00000100:00000001:2.0:1787763993.660283:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000400:00000200:0.0:1787763993.660291:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763993.660292:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763993.660297:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763993.660303:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000400:00000200:0.0:1787763993.660305:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763993.660311:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787763993.660311:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:0.0:1787763993.660318:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763993.660318:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787763993.660329:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763993.660329:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763993.660334:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763993.660335:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763993.660345:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763993.660346:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763993.660357:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763993.660359:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787763993.660361:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763993.660365:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787763993.660367:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787763993.660369:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787763993.660371:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763993.660374:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763993.660375:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763993.660377:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763993.660379:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787763993.660382:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763993.660383:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763993.660386:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787763993.660389:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763993.660391:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763993.660392:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787763993.660395:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763993.660403:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787763993.660405:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787763993.660408:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763993.660409:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763993.660412:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763993.660414:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763993.660417:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787763993.660424:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787763993.660429:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787763993.660436:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832680320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763993.660444:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763993.660448:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763993.660451:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832680320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763993.660469:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763993.660472:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787763993.660474:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787763993.660475:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763993.660477:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787763993.660479:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763993.660482:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21708
00000400:00000200:0.0:1787763993.660485:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763993.660491:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763993.660493:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787763993.660496:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763993.660498:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763993.660498:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1787763993.660512:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763993.660513:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787763993.660521:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787763993.660521:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1787763993.660528:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763993.660529:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787763993.660530:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763993.660532:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787763993.660538:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:2.0:1787763993.660539:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f0c0 x1874599832680320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787763993.660548:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787763993.660563:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:0.0:1787763993.660564:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763993.660567:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763993.660568:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1787763993.660576:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:0.0:1787763993.660577:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:2.0:1787763993.660581:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787763993.660582:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763993.660597:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:2.0:1787763993.660598:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763993.660610:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00100000:2.0:1787763993.660613:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832680320:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787763993.660616:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763993.660624:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787763993.660626:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787763993.660630:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763993.660632:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787763993.660640:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763993.660641:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763993.660646:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1787763993.660647:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:0.0:1787763993.660655:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763993.660659:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763993.660660:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
00000100:00000001:0.0:1787763993.660662:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763993.660664:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763993.660670:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000400:00000200:2.0:1787763993.660675:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:0.0:1787763993.660678:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000010:2.0:1787763993.660682:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
02000000:00000001:0.0:1787763993.660684:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763993.660689:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763993.660692:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:2.0:1787763993.660692:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
02000000:00000001:0.0:1787763993.660695:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763993.660701:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763993.660701:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832680320, portal 25
00000100:00000001:2.0:1787763993.660709:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763993.660711:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049530560 : -115096660021056 : ffff9751f876f0c0)
02000000:00000010:0.0:1787763993.660713:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000040:2.0:1787763993.660719:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f0c0 x1874599832680320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763993.660722:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763993.660734:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00100000:0.0:1787763993.660740:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832680448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:2.0:1787763993.660741:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763993.660748:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763993.660757:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832680320, offset 0
00000100:00000001:0.0:1787763993.660767:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763993.660770:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763993.660776:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787763993.660781:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832680448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787763993.660795:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787763993.660804:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787763993.660807:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787763993.660812:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00080000:0.0:1787763993.660813:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787763993.660820:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763993.660824:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1787763993.660825:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1787763993.660832:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:2.0:1787763993.660836:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00080000:0.0:1787763993.660837:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21708
00000400:00000200:2.0:1787763993.660844:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787763993.660850:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787763993.660856:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763993.660858:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1787763993.660862:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000040:0.0:1787763993.660864:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763993.660867:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1787763993.660873:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58550:58550:256:4294967295] 192.168.203.110@tcp LPNI seq info [58550:58550:8:4294967295]
00000100:00000040:0.0:1787763993.660875:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763993.660885:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787763993.660892:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:0.0:1787763993.660893:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763993.660901:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787763993.660904:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763993.660906:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787763993.660910:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787763993.660919:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:2.0:1787763993.660921:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:0.0:1787763993.660928:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763993.660931:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787763993.660935:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763993.660940:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:2.0:1787763993.660942:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000020:00000040:0.0:1787763993.660943:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787763993.660950:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763993.660952:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787763993.660953:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000010:2.0:1787763993.660954:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
02000000:00000001:0.0:1787763993.660960:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763993.660963:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763993.660964:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763993.660969:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787763993.660975:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787763993.660975:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787763993.660982:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787763993.660987:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000800:00000200:2.0:1787763993.660988:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000020:00000040:0.0:1787763993.660990:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787763993.660998:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763993.661000:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763993.661002:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787763993.661007:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:2.1:1787763993.661007:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:0.0:1787763993.661018:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787763993.661019:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1787763993.661031:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
02000000:00000010:0.0:1787763993.661032:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787763993.661042:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1787763993.661043:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000100:00100000:0.0:1787763993.661049:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832680576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787763993.661055:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1787763993.661056:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1787763993.661058:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763993.661066:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832680576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787763993.661077:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1787763993.661081:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763993.661084:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787763993.661087:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1787763993.661089:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000100:00080000:0.0:1787763993.661090:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:2.0:1787763993.661097:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00080000:0.0:1787763993.661098:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787763993.661104:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787763993.661107:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763993.661109:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1787763993.661110:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787763993.661123:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21708
00000100:00000200:2.0:1787763993.661124:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f0c0 x1874599832680320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787763993.661162:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763993.661166:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000400:00000200:0.0:1787763993.661171:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787763993.661176:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763993.661177:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f0c0 x1874599832680320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764018 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763993.661186:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763993.661198:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787763993.661203:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787763993.661206:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763993.661208:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787763993.661213:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787763993.661217:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787763993.661218:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000100:00000001:2.0:1787763993.661220:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763993.661222:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787763993.661227:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787763993.661231:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000200:2.0:1787763993.661232:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:0.0:1787763993.661238:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:2.0:1787763993.661240:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316381940
00000100:00000040:0.0:1787763993.661245:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763993.661248:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000010:2.0:1787763993.661249:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00000040:0.0:1787763993.661253:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787763993.661256:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787763993.661259:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:2.0:1787763993.661261:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00080000:0.0:1787763993.661265:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787763993.661268:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763993.661284:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316381940
02000000:00000001:0.0:1787763993.661285:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000010:2.0:1787763993.661295:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7c60.
02000000:00000001:0.0:1787763993.661296:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787763993.661302:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763993.661304:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787763993.661305:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787763993.661312:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:2.0:1787763993.661313:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000008:00000001:0.0:1787763993.661326:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763993.661327:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763993.661342:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763993.661343:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000010:0.0:1787763993.661352:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000100:00100000:2.0:1787763993.661354:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763993/real 1787763993]  req@ffff9751f876f0c0 x1874599832680320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787763993.661361:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787763993.661373:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787763993.661379:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787763993.661382:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000001:2.0:1787763993.661382:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763993.661386:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:0.0:1787763993.661387:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787763993.661393:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787763993.661393:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763993.661402:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
02000000:00000010:0.0:1787763993.661403:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000400:00000010:2.0:1787763993.661406:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:0.0:1787763993.661408:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763993.661410:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00100000:0.0:1787763993.661418:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832680704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:2.0:1787763993.661418:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f0c0 x1874599832680320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763993.661486:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787763993.661491:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787763993.661492:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f0c0 x1874599832680320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787763993.661502:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832680704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787763993.661586:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763993.661586:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763993.661594:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787763993.661595:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21718)
00000100:00000001:2.0:1787763993.661600:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763993.661603:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763993.661607:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f0c0 x1874599832680320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.661613:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763993.661616:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f0c0 x1874599832680320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.661622:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763993.661624:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763993.661625:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763993.661626:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763993.661628:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763993.661630:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763993.661632:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763993.661635:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763993.661637:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763993.661640:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f0c0 x1874599832680320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763993.661645:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832680320:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787763993.661649:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763993.661651:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763993.661653:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f0c0 x1874599832680320/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.661658:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763993.661660:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763993.661664:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763993.661666:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763993.661668:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763993.661670:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787763993.661672:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763993.661675:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017800.
02000000:00000001:2.0:1787763993.661680:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763993.661682:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763993.661684:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f0c0.
00000100:00000001:2.0:1787763993.661686:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763993.661687:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763993.661689:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763993.661695:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763993.661696:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763993.661698:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763993.661700:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763993.661701:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763993.661704:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832680448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.661709:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763993.661710:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763993.661712:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763993.661714:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763993.661715:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763993.661717:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832680448:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763993.661719:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763993.661720:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763993.661722:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763993.661723:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763993.661726:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763993.661727:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763993.661730:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763993.661732:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763993.661735:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763993.661737:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832680448, portal 10
00000100:00000001:2.0:1787763993.661739:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763993.661740:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00000040:2.0:1787763993.661746:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832680448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.661750:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763993.661752:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763993.661754:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763993.661755:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832680448, offset 0
00000400:00000200:2.0:1787763993.661758:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763993.661761:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763993.661764:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763993.661766:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763993.661768:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763993.661770:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763993.661772:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763993.661774:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58551:58551:256:4294967295] 192.168.203.110@tcp LPNI seq info [58551:58551:8:4294967295]
00000400:00000200:2.0:1787763993.661777:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763993.661782:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763993.661785:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763993.661787:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763993.661790:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763993.661792:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787763993.661795:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763993.661797:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1787763993.661800:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763993.661801:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763993.661804:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763993.661806:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763993.661808:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763993.661810:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763993.661811:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763993.661814:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e580 x1874599832680448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.661825:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763993.661827:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832680448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764018 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.661831:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763993.661832:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763993.661834:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763993.661837:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787763993.661839:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763993.661840:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316380704
00000100:00000001:2.0:1787763993.661843:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316381940
00000100:00000001:2.0:1787763993.661844:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316380704
00000100:00000001:2.0:1787763993.661846:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316381940
00000100:00000001:2.0:1787763993.661848:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763993.661849:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763993.661850:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763993.661852:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763993/real 1787763993]  req@ffff9751f876e580 x1874599832680448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763993.661857:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763993.661858:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763993.661859:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763993.661861:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763993.661862:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763993.661863:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763993.661865:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832680448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763993.661870:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832680448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.661874:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763993.661875:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763993.661877:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763993.661878:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763993.661880:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832680448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.661884:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763993.661887:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832680448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.661891:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763993.661892:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763993.661893:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763993.661894:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763993.661895:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763993.661905:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763993.661906:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763993.661908:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763993.661910:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763993.661912:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832680448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763993.661916:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832680448:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763993.661919:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763993.661922:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832680576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.661926:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763993.661928:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763993.661929:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763993.661930:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763993.661931:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763993.661934:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832680576:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763993.661936:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763993.661938:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763993.661940:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763993.661942:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763993.661946:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763993.661948:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763993.661951:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763993.661954:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763993.661957:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763993.661960:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832680576, portal 10
00000100:00000001:2.0:1787763993.661962:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763993.661964:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000100:00000040:2.0:1787763993.661970:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832680576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.661978:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763993.661981:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763993.661983:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763993.661986:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832680576, offset 0
00000400:00000200:2.0:1787763993.661990:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763993.661994:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763993.661997:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763993.661999:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763993.662001:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763993.662003:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763993.662005:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763993.662007:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58552:58552:256:4294967295] 192.168.203.110@tcp LPNI seq info [58552:58552:8:4294967295]
00000400:00000200:2.0:1787763993.662010:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763993.662014:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763993.662017:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763993.662020:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763993.662023:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763993.662025:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787763993.662028:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763993.662030:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1787763993.662032:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763993.662034:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763993.662037:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763993.662040:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763993.662044:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763993.662046:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763993.662048:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763993.662052:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e1c0 x1874599832680576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.662061:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763993.662064:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832680576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764018 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.662072:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763993.662074:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763993.662077:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763993.662080:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787763993.662084:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763993.662086:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316380692
00000100:00000001:2.0:1787763993.662089:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316381940
00000100:00000001:2.0:1787763993.662091:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316380692
00000100:00000001:2.0:1787763993.662093:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316381940
00000100:00000001:2.0:1787763993.662094:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763993.662096:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763993.662097:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763993.662100:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763993/real 1787763993]  req@ffff9751f876e1c0 x1874599832680576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763993.662105:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763993.662107:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763993.662108:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763993.662110:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763993.662111:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763993.662113:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763993.662115:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832680576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763993.662121:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832680576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.662125:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763993.662126:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763993.662127:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763993.662128:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763993.662130:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832680576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.662152:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763993.662154:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832680576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.662159:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763993.662161:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763993.662162:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763993.662163:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763993.662165:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763993.662165:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763993.662167:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763993.662169:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763993.662170:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763993.662173:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832680576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763993.662182:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832680576:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787763993.662185:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787763993.662188:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832680704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.662192:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763993.662193:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763993.662194:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787763993.662195:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763993.662197:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763993.662198:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832680704:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763993.662200:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787763993.662202:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787763993.662203:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763993.662205:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787763993.662207:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787763993.662209:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787763993.662211:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787763993.662213:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787763993.662216:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787763993.662218:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832680704, portal 4
00000100:00000001:2.0:1787763993.662219:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787763993.662220:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
00000100:00000040:2.0:1787763993.662225:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832680704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.662229:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787763993.662231:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787763993.662232:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787763993.662234:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832680704, offset 0
00000400:00000200:2.0:1787763993.662236:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787763993.662240:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763993.662243:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787763993.662245:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787763993.662247:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787763993.662249:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787763993.662251:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787763993.662253:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58553:58553:256:4294967295] 192.168.203.110@tcp LPNI seq info [58553:58553:8:4294967295]
00000400:00000200:2.0:1787763993.662255:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763993.662259:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787763993.662262:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763993.662264:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787763993.662267:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787763993.662269:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787763993.662284:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787763993.662286:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1787763993.662289:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787763993.662290:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787763993.662293:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787763993.662295:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787763993.662297:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787763993.662298:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787763993.662300:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787763993.662303:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876c000 x1874599832680704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764018 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.662308:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763993.662310:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832680704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764018 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.662315:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763993.662316:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787763993.662319:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763993.662321:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787763993.662323:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787763993.662325:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316380710
00000100:00000001:2.0:1787763993.662327:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316381940
00000100:00000001:2.0:1787763993.662329:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316380710
00000100:00000001:2.0:1787763993.662331:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316381940
00000100:00000001:2.0:1787763993.662332:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787763993.662333:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763993.662334:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787763993.662337:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763993/real 1787763993]  req@ffff9751f876c000 x1874599832680704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787763993.662341:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787763993.662342:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787763993.662344:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787763993.662345:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787763993.662346:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787763993.662348:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787763993.662350:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832680704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787763993.662355:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832680704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.662359:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787763993.662360:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763993.662361:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763993.662362:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787763993.662364:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832680704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.662368:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787763993.662370:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832680704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.662374:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787763993.662376:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763993.662377:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787763993.662378:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763993.662380:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787763993.662381:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787763993.662383:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787763993.662385:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787763993.662386:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787763993.662388:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832680704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787763993.662393:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832680704:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787763993.662395:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763993.662396:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763993.662398:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832680448/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.662402:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763993.662403:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763993.662405:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787763993.662406:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763993.662407:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763993.662408:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763993.662410:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763993.662411:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016c00.
02000000:00000001:2.0:1787763993.662414:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763993.662415:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763993.662416:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:2.0:1787763993.662420:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763993.662421:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763993.662422:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763993.662424:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832680576/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.662434:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763993.662435:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763993.662436:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787763993.662438:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763993.662438:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763993.662439:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787763993.662440:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763993.662441:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
02000000:00000001:2.0:1787763993.662442:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763993.662443:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763993.662444:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:2.0:1787763993.662445:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763993.662446:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763993.662446:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787763993.662448:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832680704/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764018 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763993.662452:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787763993.662453:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787763993.662453:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787763993.662455:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787763993.662455:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787763993.662456:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787763993.662457:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787763993.662458:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:2.0:1787763993.662459:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787763993.662460:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787763993.662461:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:2.0:1787763993.662463:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787763993.662464:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763993.662465:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763993.662467:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763993.662468:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763993.662469:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787763993.662470:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763994.300253:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763994.300256:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763994.300261:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763994.300263:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:1.0:1787763994.300264:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763994.300278:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763994.300283:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763994.300285:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763994.300288:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:1.0:1787763994.300290:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763994.300292:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763994.300296:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763994.300296:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763994.300303:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763994.300304:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763994.300306:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763994.300311:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763994.300313:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763994.300317:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763994.300323:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763994.300325:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763994.300376:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763994.300382:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763994.300385:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763994.684293:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763994.684305:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763994.684308:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763994.684315:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763994.684320:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763994.684324:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763994.684328:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763994.684331:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763994.684335:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763994.940282:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316381940, at 4316381952
00000100:00000010:3.0:1787763994.940297:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7c60.
00000100:00000040:2.0:1787763994.940350:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316381940, at 4316381952
00000100:00000010:2.0:1787763994.940362:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00000100:00000001:1.0:1787763995.324221:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763995.324227:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763995.324228:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787763995.324231:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763995.324233:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763995.324236:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763995.324242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763995.324243:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763995.324247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763995.324249:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763995.324253:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763995.324256:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763995.324257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763995.324272:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763995.324275:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763995.324326:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763995.324330:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763995.324333:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763995.708513:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763995.708523:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763995.708526:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763995.708533:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763995.708543:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763995.708550:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763995.708553:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763995.708556:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763995.708560:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763996.348213:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763996.348213:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763996.348221:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763996.348224:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763996.348228:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763996.348229:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763996.348232:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00100000:1.0:1787763996.348236:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763996.348236:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787763996.348239:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763996.348242:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763996.348244:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787763996.348247:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763996.348249:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763996.348250:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763996.348265:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763996.348266:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763996.348270:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763996.348272:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763996.348274:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763996.348276:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763996.348381:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763996.348389:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763996.348393:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763996.732276:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763996.732288:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763996.732291:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763996.732299:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763996.732304:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763996.732309:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763996.732313:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763996.732317:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763996.732321:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763997.372267:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763997.372268:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763997.372275:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763997.372277:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763997.372281:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763997.372283:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763997.372286:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763997.372288:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763997.372289:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763997.372298:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787763997.372303:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763997.372307:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763997.372310:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763997.372312:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763997.372315:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763997.372354:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763997.372359:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763997.372362:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763997.756232:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763997.756267:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763997.756271:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763997.756279:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763997.756285:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763997.756289:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763997.756293:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763997.756298:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763997.756303:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763998.396248:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763998.396254:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763998.396257:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763998.396259:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763998.396262:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763998.396270:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763998.396271:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763998.396276:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787763998.396276:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763998.396282:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787763998.396283:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763998.396287:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787763998.396287:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763998.396292:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763998.396295:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763998.396341:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763998.396346:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763998.396349:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763998.780272:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763998.780280:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763998.780282:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763998.780288:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763998.780292:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763998.780296:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763998.780300:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763998.780303:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763998.780306:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763998.781389:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763998.781396:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787763998.781399:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763998.781402:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763998.781407:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763998.781415:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21713
00000400:00000200:2.0:1787763998.781423:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763998.781431:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763998.781435:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763998.781439:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763998.781442:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763998.781445:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763998.781447:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763998.781451:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763998.781453:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787763998.781455:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763998.781459:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787763998.781461:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787763998.781464:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787763998.781467:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763998.781469:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763998.781471:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763998.781473:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763998.781477:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787763998.781481:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763998.781484:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763998.781487:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763998.781490:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763998.781493:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763998.781495:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787763998.781498:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763998.781506:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658780.
00000020:00000040:2.0:1787763998.781509:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787763998.781514:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763998.781515:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763998.781518:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763998.781521:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763998.781525:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763998.781534:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bb800.
00000100:00000001:2.0:1787763998.781540:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763998.781549:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658780 x1874599832680832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763998.781560:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763998.781565:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763998.781570:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000744875b5] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658780 x1874599832680832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763998.781604:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763998.781608:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763998.781612:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787763998.781613:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763998.781615:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763998.781619:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763998.781623:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21713
00000400:00000200:2.0:1787763998.781628:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763998.781634:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763998.781636:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763998.781640:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763998.781642:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763998.781644:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763998.781646:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763998.781648:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763998.781649:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:2.0:1787763998.781651:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763998.781653:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:2.0:1787763998.781655:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:2.0:1787763998.781657:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:2.0:1787763998.781660:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763998.781662:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763998.781664:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763998.781666:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763998.781669:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:2.0:1787763998.781672:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763998.781674:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763998.781676:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763998.781681:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763998.781683:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763998.781685:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787763998.781689:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787763998.781692:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763998.781695:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763998.781700:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000020:00000040:2.0:1787763998.781703:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787763998.781705:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763998.781707:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763998.781709:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763998.781711:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763998.781713:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763998.781718:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5b8c00.
00000100:00000001:2.0:1787763998.781722:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763998.781727:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832680960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763998.781735:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763998.781737:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763998.781741:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832680960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763998.781750:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787763998.781752:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763998.781756:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1787763998.781758:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763998.781759:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763998.781762:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763998.781765:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21713
00000400:00000200:2.0:1787763998.781769:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763998.781774:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763998.781777:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763998.781780:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763998.781782:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763998.781784:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763998.781786:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763998.781788:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763998.781790:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787763998.781792:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763998.781795:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:2.0:1787763998.781797:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:2.0:1787763998.781799:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:2.0:1787763998.781803:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763998.781806:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763998.781809:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763998.781811:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763998.781814:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:2.0:1787763998.781817:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763998.781819:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763998.781821:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763998.781824:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763998.781826:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763998.781828:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787763998.781830:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787763998.781832:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763998.781834:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763998.781838:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000020:00000040:2.0:1787763998.781841:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787763998.781843:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763998.781845:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763998.781847:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763998.781848:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763998.781850:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763998.781855:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5b9800.
00000100:00000001:2.0:1787763998.781858:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763998.781862:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832681088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763998.781870:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763998.781872:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763998.781876:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832681088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763998.781885:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763998.781887:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1787763998.781892:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:2.0:1787763998.781895:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787763998.781898:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787763998.781899:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787763998.781901:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787763998.781904:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787763998.781907:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21713
00000400:00000200:2.0:1787763998.781910:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787763998.781913:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787763998.781915:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787763998.781918:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763998.781920:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763998.781922:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763998.781924:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787763998.781927:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787763998.781928:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787763998.781930:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787763998.781932:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787763998.781934:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:2.0:1787763998.781936:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:2.0:1787763998.781939:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763998.781941:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787763998.781942:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787763998.781944:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787763998.781946:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:2.0:1787763998.781949:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787763998.781951:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787763998.781953:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787763998.781957:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787763998.781959:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763998.781961:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787763998.781964:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1787763998.781966:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763998.781968:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787763998.781972:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
00000020:00000040:2.0:1787763998.781975:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:2.0:1787763998.781977:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787763998.781978:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787763998.781979:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787763998.781980:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787763998.781981:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787763998.781984:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5ba000.
00000100:00000001:2.0:1787763998.781987:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763998.781990:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832681216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787763998.781995:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787763998.781997:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763998.782000:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832681216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787763998.782005:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787763998.782008:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21723)
00000100:00000001:3.0:1787763998.782071:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763998.782076:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787763998.782078:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763998.782085:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832680832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782090:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763998.782092:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763998.782094:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763998.782096:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763998.782098:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763998.782101:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832680832:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787763998.782104:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763998.782108:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763998.782109:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763998.782111:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763998.782115:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787763998.782116:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763998.782120:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763998.782124:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763998.782129:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763998.782169:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832680832, portal 25
00000100:00000001:3.0:1787763998.782177:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763998.782179:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000040:3.0:1787763998.782187:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832680832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782195:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763998.782199:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763998.782203:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763998.782207:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832680832, offset 0
00000400:00000200:3.0:1787763998.782215:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763998.782225:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763998.782232:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763998.782247:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763998.782251:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763998.782260:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763998.782262:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763998.782265:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58554:58554:256:4294967295] 192.168.203.110@tcp LPNI seq info [58554:58554:8:4294967295]
00000400:00000200:3.0:1787763998.782269:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763998.782277:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763998.782281:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763998.782286:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763998.782291:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763998.782294:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787763998.782298:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763998.782301:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787763998.782308:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763998.782310:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763998.782313:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763998.782317:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763998.782320:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763998.782322:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763998.782326:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763998.782329:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832680832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782336:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763998.782338:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832680832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764023 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782343:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763998.782344:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763998.782349:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763998.782352:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763998.782354:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316387061
00000100:00000010:3.0:1787763998.782357:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40ae0.
00000100:00000001:3.0:1787763998.782364:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787763998.782369:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316387061
00000100:00000010:3.0:1787763998.782371:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40d20.
00000100:00000001:3.0:1787763998.782373:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787763998.782375:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763998.782377:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763998.782379:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763998.782382:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763998/real 1787763998]  req@ffff9751f8658780 x1874599832680832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763998.782387:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763998.782389:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763998.782390:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763998.782392:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763998.782393:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763998.782395:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763998.782397:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832680832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763998.782402:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832680832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782406:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763998.782407:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763998.782409:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763998.782410:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763998.782413:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832680832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782417:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763998.782420:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832680832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782425:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763998.782427:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763998.782429:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763998.782430:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763998.782432:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763998.782433:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763998.782435:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763998.782438:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763998.782439:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763998.782442:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832680832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763998.782447:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832680832:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787763998.782450:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763998.782452:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832680960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782457:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763998.782458:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763998.782459:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763998.782461:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763998.782462:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763998.782464:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832680960:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763998.782465:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763998.782467:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763998.782468:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763998.782469:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763998.782471:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787763998.782473:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763998.782475:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763998.782476:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763998.782479:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763998.782481:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832680960, portal 10
00000100:00000001:3.0:1787763998.782482:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763998.782483:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:3.0:1787763998.782488:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832680960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782492:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763998.782494:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763998.782495:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763998.782497:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832680960, offset 0
00000400:00000200:3.0:1787763998.782499:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763998.782501:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763998.782504:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763998.782506:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763998.782507:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763998.782509:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763998.782511:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763998.782513:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58555:58555:256:4294967295] 192.168.203.110@tcp LPNI seq info [58555:58555:8:4294967295]
00000400:00000200:3.0:1787763998.782515:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763998.782519:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763998.782522:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763998.782524:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763998.782526:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763998.782528:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787763998.782531:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763998.782533:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787763998.782535:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763998.782537:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763998.782539:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763998.782541:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763998.782543:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763998.782544:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763998.782546:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763998.782548:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832680960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782553:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763998.782555:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832680960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764023 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782560:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763998.782561:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763998.782563:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763998.782565:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787763998.782567:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763998.782568:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316385825
00000100:00000001:3.0:1787763998.782570:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316387061
00000100:00000001:3.0:1787763998.782572:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316385825
00000100:00000001:3.0:1787763998.782573:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316387061
00000100:00000001:3.0:1787763998.782575:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763998.782576:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763998.782577:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763998.782579:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763998/real 1787763998]  req@ffff9751f865ad00 x1874599832680960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763998.782584:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763998.782585:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763998.782586:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763998.782587:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763998.782588:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763998.782589:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763998.782591:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832680960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763998.782596:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832680960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782600:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763998.782601:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763998.782602:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763998.782603:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763998.782606:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832680960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782610:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763998.782613:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832680960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782617:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763998.782618:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763998.782619:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763998.782620:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763998.782621:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763998.782631:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763998.782633:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763998.782636:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763998.782638:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763998.782642:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832680960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763998.782650:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832680960:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763998.782654:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763998.782657:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832681088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782664:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763998.782665:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763998.782667:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763998.782669:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763998.782671:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763998.782674:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832681088:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763998.782678:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763998.782680:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763998.782681:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763998.782683:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763998.782687:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610c00.
02000000:00000001:3.0:1787763998.782689:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763998.782693:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763998.782696:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763998.782698:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763998.782701:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832681088, portal 10
00000100:00000001:3.0:1787763998.782703:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763998.782704:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:3.0:1787763998.782710:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832681088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782715:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763998.782717:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763998.782718:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763998.782720:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832681088, offset 0
00000400:00000200:3.0:1787763998.782723:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763998.782726:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763998.782728:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763998.782730:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763998.782732:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763998.782734:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763998.782735:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763998.782737:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58556:58556:256:4294967295] 192.168.203.110@tcp LPNI seq info [58556:58556:8:4294967295]
00000400:00000200:3.0:1787763998.782740:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763998.782744:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763998.782747:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763998.782749:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763998.782752:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763998.782754:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787763998.782757:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763998.782758:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787763998.782761:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763998.782763:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763998.782765:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763998.782767:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763998.782769:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763998.782771:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763998.782772:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763998.782775:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832681088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782780:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763998.782782:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832681088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764023 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782786:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763998.782787:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763998.782789:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763998.782791:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787763998.782793:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763998.782795:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316385813
00000100:00000001:3.0:1787763998.782796:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316387061
00000100:00000001:3.0:1787763998.782798:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316385813
00000100:00000001:3.0:1787763998.782799:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316387061
00000100:00000001:3.0:1787763998.782801:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763998.782802:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763998.782803:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763998.782805:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763998/real 1787763998]  req@ffff9751f8659a40 x1874599832681088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763998.782810:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763998.782811:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763998.782812:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763998.782814:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763998.782815:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763998.782816:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763998.782818:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832681088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763998.782824:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832681088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782827:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763998.782828:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763998.782829:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763998.782830:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763998.782832:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832681088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782836:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763998.782838:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832681088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782842:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763998.782844:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763998.782845:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763998.782846:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763998.782847:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763998.782848:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763998.782849:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763998.782852:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763998.782853:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763998.782856:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832681088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763998.782861:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832681088:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787763998.782863:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787763998.782865:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832681216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782870:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787763998.782871:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763998.782872:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787763998.782873:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787763998.782875:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787763998.782876:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832681216:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763998.782878:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787763998.782879:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787763998.782880:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787763998.782881:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787763998.782884:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787763998.782885:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787763998.782887:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787763998.782889:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787763998.782891:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787763998.782893:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832681216, portal 4
00000100:00000001:3.0:1787763998.782894:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787763998.782895:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:3.0:1787763998.782899:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832681216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782903:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787763998.782904:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787763998.782906:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787763998.782907:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832681216, offset 0
00000400:00000200:3.0:1787763998.782909:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787763998.782911:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763998.782913:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787763998.782915:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787763998.782916:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787763998.782918:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787763998.782919:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787763998.782922:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58557:58557:256:4294967295] 192.168.203.110@tcp LPNI seq info [58557:58557:8:4294967295]
00000400:00000200:3.0:1787763998.782924:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787763998.782929:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787763998.782932:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763998.782933:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787763998.782936:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787763998.782937:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787763998.782940:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787763998.782942:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787763998.782944:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787763998.782946:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787763998.782948:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787763998.782950:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787763998.782952:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787763998.782953:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787763998.782954:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787763998.782956:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832681216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764023 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782961:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763998.782963:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832681216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764023 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.782967:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763998.782969:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787763998.782970:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787763998.782972:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787763998.782974:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787763998.782975:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316385830
00000100:00000001:3.0:1787763998.782977:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316387061
00000100:00000001:3.0:1787763998.782978:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316385830
00000100:00000001:3.0:1787763998.782980:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316387061
00000100:00000001:3.0:1787763998.782981:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787763998.782982:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763998.782984:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787763998.782986:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787763998/real 1787763998]  req@ffff9751f8658000 x1874599832681216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787763998.782991:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787763998.782992:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787763998.782992:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787763998.782994:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787763998.782995:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787763998.782996:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787763998.782998:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832681216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787763998.783003:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832681216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.783007:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787763998.783008:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763998.783009:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763998.783010:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787763998.783013:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832681216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.783019:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787763998.783023:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832681216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.783030:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787763998.783032:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763998.783033:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787763998.783035:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763998.783037:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787763998.783038:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787763998.783041:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787763998.783045:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787763998.783047:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787763998.783051:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832681216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787763998.783060:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832681216:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787763998.783064:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763998.783066:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763998.783069:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832680832/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.783076:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763998.783079:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763998.783081:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787763998.783084:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763998.783086:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763998.783088:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787763998.783090:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763998.783093:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:3.0:1787763998.783096:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763998.783097:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763998.783100:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:3.0:1787763998.783102:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763998.783103:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763998.783105:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763998.783108:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832680960/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.783114:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763998.783116:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763998.783117:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787763998.783119:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763998.783121:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763998.783122:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763998.783124:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763998.783126:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:3.0:1787763998.783128:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763998.783129:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763998.783131:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:3.0:1787763998.783156:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763998.783158:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763998.783160:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763998.783164:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832681088/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.783170:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763998.783171:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763998.783172:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610c00.
02000000:00000001:3.0:1787763998.783175:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763998.783176:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763998.783177:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787763998.783179:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763998.783181:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:3.0:1787763998.783183:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763998.783184:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763998.783186:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:3.0:1787763998.783188:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763998.783189:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763998.783191:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787763998.783194:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832681216/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764023 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787763998.783200:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787763998.783201:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787763998.783203:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787763998.783205:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787763998.783207:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787763998.783208:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787763998.783210:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787763998.783211:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:3.0:1787763998.783213:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787763998.783214:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787763998.783216:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:3.0:1787763998.783218:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787763998.783219:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763998.783222:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787763998.783228:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787763998.783230:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787763998.783232:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787763998.783246:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763999.420223:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787763999.420223:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763999.420229:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763999.420244:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787763999.420247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787763999.420249:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787763999.420250:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787763999.420257:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787763999.420257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787763999.420258:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787763999.420265:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787763999.420269:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763999.420272:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787763999.420274:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787763999.420277:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787763999.420317:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787763999.420322:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787763999.420326:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787763999.804205:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787763999.804213:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787763999.804217:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787763999.804223:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787763999.804244:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787763999.804248:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763999.804252:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787763999.804255:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787763999.804259:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764000.060184:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316387061, at 4316387072
00000100:00000010:3.0:1787764000.060207:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40d20.
00000100:00000040:2.0:1787764000.060260:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316387061, at 4316387072
00000100:00000010:2.0:1787764000.060272:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40ae0.
00000100:00000001:1.0:1787764000.444288:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764000.444297:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764000.444300:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764000.444302:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764000.444305:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764000.444310:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764000.444313:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764000.444320:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764000.444324:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764000.444325:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764000.444328:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764000.444331:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764000.444333:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764000.444334:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764000.444342:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764000.444404:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764000.444408:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764000.444411:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764000.828254:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764000.828260:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764000.828263:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764000.828268:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764000.828273:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764000.828277:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764000.828280:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764000.828282:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764000.828285:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764001.468209:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764001.468230:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764001.468232:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764001.468236:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764001.468237:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764001.468240:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764001.468242:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764001.468246:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764001.468246:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764001.468250:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764001.468250:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764001.468253:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764001.468255:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764001.468256:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764001.468259:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764001.468262:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764001.468265:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764001.468267:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764001.468269:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764001.468270:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764001.468272:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764001.468316:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764001.468319:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764001.468322:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764001.852253:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764001.852263:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764001.852266:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764001.852272:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764001.852277:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764001.852281:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764001.852284:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764001.852286:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764001.852289:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764002.492221:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764002.492224:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764002.492229:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764002.492230:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764002.492235:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764002.492235:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764002.492239:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764002.492242:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764002.492244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764002.492246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764002.492250:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764002.492255:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764002.492259:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764002.492261:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764002.492264:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764002.492289:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764002.492293:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764002.492296:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764002.876269:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764002.876278:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764002.876281:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764002.876286:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764002.876291:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764002.876294:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764002.876296:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764002.876299:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764002.876302:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764003.516236:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764003.516243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764003.516246:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764003.516247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764003.516250:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764003.516253:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764003.516255:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764003.516257:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764003.516259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764003.516261:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764003.516264:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764003.516268:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764003.516271:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764003.516273:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764003.516276:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764003.516309:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764003.516312:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764003.516315:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764003.900244:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764003.900251:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764003.900253:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764003.900255:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1787764003.900258:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764003.900264:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1787764003.900264:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764003.900268:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787764003.900269:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1787764003.900272:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764003.900273:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764003.900276:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764003.900278:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787764003.900280:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764003.900285:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21718
00000400:00000200:3.0:1787764003.900292:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764003.900297:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764003.900300:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764003.900302:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764003.900305:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764003.900306:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764003.900308:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764003.900311:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764003.900313:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787764003.900315:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764003.900318:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787764003.900319:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787764003.900321:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787764003.900323:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764003.900324:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764003.900325:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764003.900326:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764003.900329:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787764003.900331:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764003.900333:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764003.900336:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764003.900338:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764003.900340:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764003.900341:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764003.900343:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764003.900351:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bf480.
00000020:00000040:3.0:1787764003.900355:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787764003.900359:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764003.900361:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764003.900364:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764003.900366:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764003.900370:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764003.900378:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611c00.
00000100:00000001:3.0:1787764003.900385:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764003.900394:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bf480 x1874599832681344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764003.900404:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764003.900408:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764003.900411:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c2965388] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bf480 x1874599832681344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764003.900430:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764003.900433:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764003.900435:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787764003.900436:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764003.900437:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764003.900439:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764003.900442:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21718
00000400:00000200:3.0:1787764003.900445:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764003.900448:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764003.900450:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787764003.900452:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1787764003.900453:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764003.900457:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764003.900457:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764003.900459:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764003.900461:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1787764003.900464:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764003.900467:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764003.900469:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00000040:2.0:1787764003.900470:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bf480 x1874599832681344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787764003.900471:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764003.900474:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:3.0:1787764003.900479:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:3.0:1787764003.900483:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787764003.900486:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764003.900489:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764003.900489:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764003.900493:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1787764003.900494:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1787764003.900498:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1787764003.900498:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764003.900503:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764003.900504:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764003.900508:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00100000:2.0:1787764003.900510:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832681344:192.168.203.110@tcp:250:kworker.0
00000100:00080000:3.0:1787764003.900513:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764003.900518:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1787764003.900518:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764003.900522:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764003.900523:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764003.900526:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764003.900527:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764003.900528:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:3.0:1787764003.900531:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764003.900534:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000010:2.0:1787764003.900536:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
00000008:00100000:3.0:1787764003.900537:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1787764003.900541:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1787764003.900542:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764003.900545:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764003.900549:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000010:3.0:1787764003.900550:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc000.
00000400:00000010:2.0:1787764003.900553:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000020:00000040:3.0:1787764003.900554:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787764003.900560:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764003.900561:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764003.900563:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000001:3.0:1787764003.900563:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764003.900564:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764003.900569:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764003.900572:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832681344, portal 25
02000000:00000010:3.0:1787764003.900574:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610c00.
00000100:00000001:2.0:1787764003.900576:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764003.900578:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049859200 : -115096659692416 : ffff9751f87bf480)
00000100:00000001:3.0:1787764003.900581:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764003.900586:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bf480 x1874599832681344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764003.900589:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc000 x1874599832681472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764003.900605:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1787764003.900608:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:2.0:1787764003.900612:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787764003.900613:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764003.900617:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000040:3.0:1787764003.900619:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3fc1c6c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc000 x1874599832681472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764003.900621:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832681344, offset 0
00000100:00000001:3.0:1787764003.900632:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764003.900635:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00080000:3.0:1787764003.900636:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764003.900639:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1787764003.900640:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764003.900642:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1787764003.900643:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764003.900646:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764003.900650:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21718
00000400:00000200:2.0:1787764003.900653:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764003.900658:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764003.900661:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764003.900664:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764003.900666:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764003.900669:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58558:58558:256:4294967295] 192.168.203.110@tcp LPNI seq info [58558:58558:8:4294967295]
00000400:00000200:2.0:1787764003.900672:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764003.900678:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000400:00000200:3.0:1787764003.900686:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:2.0:1787764003.900687:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764003.900691:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000800:00000010:2.0:1787764003.900692:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000100:00080000:3.0:1787764003.900693:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764003.900698:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764003.900700:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1787764003.900704:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:3.0:1787764003.900705:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764003.900709:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:2.0:1787764003.900710:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:3.0:1787764003.900712:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764003.900716:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:3.0:1787764003.900717:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764003.900721:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:3.0:1787764003.900723:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:3.0:1787764003.900724:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:3.0:1787764003.900726:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764003.900728:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.1:1787764003.900728:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:3.0:1787764003.900733:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764003.900736:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:2.1:1787764003.900736:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000020:00000040:3.0:1787764003.900739:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:3.0:1787764003.900745:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000800:00000100:2.0:1787764003.900746:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764003.900750:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000100:00000001:3.0:1787764003.900752:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764003.900756:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:3.0:1787764003.900756:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764003.900762:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1787764003.900764:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:3.0:1787764003.900767:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764003.900770:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000100:00000001:3.0:1787764003.900772:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:2.0:1787764003.900773:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000008:00100000:3.0:1787764003.900777:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1787764003.900778:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000008:00000001:3.0:1787764003.900784:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764003.900786:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bf480 x1874599832681344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764003.900788:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764003.900796:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87be1c0.
00000020:00000040:3.0:1787764003.900802:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000001:2.0:1787764003.900805:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:3.0:1787764003.900805:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764003.900809:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764003.900812:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:2.0:1787764003.900813:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bf480 x1874599832681344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764028 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787764003.900814:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764003.900817:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764003.900823:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610800.
00000100:00000001:3.0:1787764003.900828:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764003.900830:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764003.900835:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87be1c0 x1874599832681600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764003.900836:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764003.900844:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764003.900845:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764003.900847:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764003.900852:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:3.0:1787764003.900853:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e2011c6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87be1c0 x1874599832681600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.900856:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316392179
00000100:00000001:3.0:1787764003.900864:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764003.900866:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1787764003.900868:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000010:2.0:1787764003.900870:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00080000:3.0:1787764003.900871:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764003.900877:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787764003.900878:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787764003.900880:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00080000:3.0:1787764003.900880:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764003.900885:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764003.900887:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21718
00000100:00000001:2.0:1787764003.900890:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316392179
00000400:00000200:3.0:1787764003.900893:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000010:2.0:1787764003.900895:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7ea0.
00000400:00000200:3.0:1787764003.900898:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787764003.900900:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00080000:3.0:1787764003.900901:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787764003.900905:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1787764003.900907:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764003.900910:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764003.900911:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764003.900914:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000040:3.0:1787764003.900914:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764003.900918:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00100000:2.0:1787764003.900922:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764003/real 1787764003]  req@ffff9751f87bf480 x1874599832681344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787764003.900922:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764003.900925:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:3.0:1787764003.900929:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764003.900932:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:3.0:1787764003.900936:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000001:2.0:1787764003.900938:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000001:3.0:1787764003.900939:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000001:2.0:1787764003.900942:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:3.0:1787764003.900945:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000010:2.0:1787764003.900946:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000100:00000001:3.0:1787764003.900950:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1787764003.900951:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787764003.900955:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000040:3.0:1787764003.900955:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764003.900960:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764003.900961:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000040:3.0:1787764003.900964:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000200:2.0:1787764003.900967:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bf480 x1874599832681344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787764003.900970:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764003.900975:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764003.900979:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764003.900983:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764003.900986:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764003.900989:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bf480 x1874599832681344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764003.900989:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787764003.900992:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1787764003.900998:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764003.901002:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764003.901008:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bed00.
00000100:00000001:2.0:1787764003.901010:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000020:00000040:3.0:1787764003.901011:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:2.0:1787764003.901014:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787764003.901015:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764003.901017:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764003.901020:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1787764003.901021:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:3.0:1787764003.901022:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787764003.901026:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787764003.901026:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764003.901032:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612000.
00000100:00100000:2.0:1787764003.901036:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bf480 x1874599832681344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764003.901037:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764003.901046:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bed00 x1874599832681728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764003.901048:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764003.901057:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:2.0:1787764003.901058:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bf480 x1874599832681344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764003.901061:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764003.901070:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000079751c9a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bed00 x1874599832681728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901071:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764003.901076:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764003.901080:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764003.901081:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764003.901085:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764003.901088:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764003.901093:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:3.0:1787764003.901093:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764003.901100:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764003.901103:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764003.901103:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21728)
00000100:00000040:2.0:1787764003.901111:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bf480 x1874599832681344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764003.901117:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832681344:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764003.901121:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764003.901123:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764003.901126:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bf480 x1874599832681344/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901131:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764003.901132:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764003.901150:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764003.901153:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764003.901154:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764003.901155:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764003.901157:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764003.901159:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611c00.
02000000:00000001:2.0:1787764003.901161:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764003.901163:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764003.901164:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bf480.
00000100:00000001:2.0:1787764003.901166:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764003.901167:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764003.901169:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764003.901174:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764003.901175:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764003.901176:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764003.901177:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764003.901178:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764003.901181:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc000 x1874599832681472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901186:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764003.901187:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764003.901189:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764003.901190:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764003.901192:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764003.901194:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832681472:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764003.901196:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764003.901197:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764003.901198:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764003.901216:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764003.901219:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764003.901221:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764003.901226:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764003.901229:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787764003.901232:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787764003.901235:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832681472, portal 10
00000100:00000001:2.0:1787764003.901238:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764003.901239:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049845760 : -115096659705856 : ffff9751f87bc000)
00000100:00000040:2.0:1787764003.901246:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc000 x1874599832681472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901254:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764003.901256:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764003.901259:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787764003.901261:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832681472, offset 0
00000400:00000200:2.0:1787764003.901266:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764003.901270:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764003.901274:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764003.901277:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764003.901279:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764003.901282:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764003.901284:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764003.901287:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58559:58559:256:4294967295] 192.168.203.110@tcp LPNI seq info [58559:58559:8:4294967295]
00000400:00000200:2.0:1787764003.901292:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764003.901299:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764003.901304:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764003.901308:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787764003.901313:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764003.901316:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787764003.901321:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764003.901324:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1787764003.901328:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764003.901330:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787764003.901332:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764003.901335:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764003.901337:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787764003.901338:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787764003.901340:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764003.901343:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc000 x1874599832681472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901349:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764003.901351:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc000 x1874599832681472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764028 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901355:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764003.901357:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764003.901359:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764003.901362:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764003.901365:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764003.901366:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316390944
00000100:00000001:2.0:1787764003.901368:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316392179
00000100:00000001:2.0:1787764003.901370:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316390944
00000100:00000001:2.0:1787764003.901371:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316392179
00000100:00000001:2.0:1787764003.901373:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764003.901375:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764003.901376:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764003.901378:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764003/real 1787764003]  req@ffff9751f87bc000 x1874599832681472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764003.901385:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764003.901386:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764003.901387:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787764003.901389:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787764003.901390:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787764003.901391:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764003.901393:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc000 x1874599832681472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764003.901399:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc000 x1874599832681472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901403:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764003.901404:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764003.901405:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764003.901406:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764003.901408:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc000 x1874599832681472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901412:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764003.901414:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc000 x1874599832681472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901418:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764003.901420:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764003.901421:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764003.901422:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764003.901423:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764003.901433:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764003.901435:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764003.901437:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764003.901439:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764003.901442:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc000 x1874599832681472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764003.901448:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832681472:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764003.901450:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764003.901452:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87be1c0 x1874599832681600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901456:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764003.901457:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764003.901458:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764003.901459:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764003.901461:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764003.901462:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832681600:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764003.901464:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764003.901466:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764003.901467:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764003.901468:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764003.901470:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764003.901471:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764003.901474:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764003.901475:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787764003.901478:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787764003.901480:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832681600, portal 10
00000100:00000001:2.0:1787764003.901481:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764003.901482:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049854400 : -115096659697216 : ffff9751f87be1c0)
00000100:00000040:2.0:1787764003.901486:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87be1c0 x1874599832681600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901491:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764003.901492:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764003.901494:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787764003.901495:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832681600, offset 0
00000400:00000200:2.0:1787764003.901498:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764003.901501:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764003.901503:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764003.901505:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764003.901507:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764003.901508:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764003.901510:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764003.901512:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58560:58560:256:4294967295] 192.168.203.110@tcp LPNI seq info [58560:58560:8:4294967295]
00000400:00000200:2.0:1787764003.901515:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764003.901519:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764003.901522:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764003.901524:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787764003.901526:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764003.901528:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787764003.901531:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764003.901533:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1787764003.901535:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764003.901537:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787764003.901538:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764003.901540:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764003.901542:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787764003.901543:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787764003.901545:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764003.901547:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87be1c0 x1874599832681600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901552:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764003.901554:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87be1c0 x1874599832681600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764028 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901558:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764003.901559:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764003.901560:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764003.901562:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764003.901564:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764003.901566:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316390932
00000100:00000001:2.0:1787764003.901567:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316392179
00000100:00000001:2.0:1787764003.901569:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316390932
00000100:00000001:2.0:1787764003.901570:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316392179
00000100:00000001:2.0:1787764003.901571:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764003.901572:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764003.901574:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764003.901576:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764003/real 1787764003]  req@ffff9751f87be1c0 x1874599832681600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764003.901580:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764003.901581:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764003.901582:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787764003.901583:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787764003.901584:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787764003.901586:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764003.901588:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87be1c0 x1874599832681600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764003.901593:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87be1c0 x1874599832681600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901597:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764003.901598:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764003.901599:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764003.901600:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764003.901602:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87be1c0 x1874599832681600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901606:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764003.901608:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87be1c0 x1874599832681600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901612:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764003.901614:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764003.901615:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764003.901615:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764003.901617:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764003.901618:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764003.901619:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764003.901621:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764003.901622:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764003.901625:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87be1c0 x1874599832681600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764003.901629:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832681600:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764003.901632:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764003.901633:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bed00 x1874599832681728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901637:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764003.901638:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764003.901639:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764003.901640:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764003.901642:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764003.901643:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832681728:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764003.901645:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764003.901646:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764003.901647:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764003.901648:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764003.901651:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764003.901652:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764003.901654:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764003.901655:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787764003.901657:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787764003.901659:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832681728, portal 4
00000100:00000001:2.0:1787764003.901661:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764003.901662:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049857280 : -115096659694336 : ffff9751f87bed00)
00000100:00000040:2.0:1787764003.901665:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bed00 x1874599832681728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901669:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764003.901671:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764003.901672:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787764003.901673:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832681728, offset 0
00000400:00000200:2.0:1787764003.901676:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764003.901678:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764003.901680:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764003.901681:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764003.901683:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764003.901684:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764003.901686:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764003.901688:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58561:58561:256:4294967295] 192.168.203.110@tcp LPNI seq info [58561:58561:8:4294967295]
00000400:00000200:2.0:1787764003.901690:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764003.901694:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764003.901697:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764003.901699:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787764003.901701:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764003.901702:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787764003.901705:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764003.901706:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1787764003.901709:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764003.901710:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787764003.901712:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764003.901714:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764003.901716:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787764003.901717:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787764003.901718:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764003.901721:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bed00 x1874599832681728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764028 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901730:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764003.901732:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bed00 x1874599832681728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764028 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901736:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764003.901737:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764003.901739:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764003.901740:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764003.901742:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764003.901743:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316390949
00000100:00000001:2.0:1787764003.901744:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316392179
00000100:00000001:2.0:1787764003.901746:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316390949
00000100:00000001:2.0:1787764003.901747:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316392179
00000100:00000001:2.0:1787764003.901749:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764003.901750:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764003.901751:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764003.901753:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764003/real 1787764003]  req@ffff9751f87bed00 x1874599832681728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764003.901757:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764003.901758:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764003.901759:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787764003.901760:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787764003.901761:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787764003.901762:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764003.901764:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bed00 x1874599832681728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764003.901769:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bed00 x1874599832681728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901773:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764003.901774:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764003.901776:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764003.901777:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764003.901779:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bed00 x1874599832681728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901783:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764003.901787:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bed00 x1874599832681728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901794:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764003.901797:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764003.901799:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764003.901800:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764003.901802:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764003.901803:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764003.901806:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764003.901809:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764003.901812:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764003.901816:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bed00 x1874599832681728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764003.901825:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832681728:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764003.901829:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764003.901831:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764003.901834:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc000 x1874599832681472/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901841:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764003.901843:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764003.901844:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764003.901846:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764003.901847:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764003.901848:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764003.901850:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764003.901851:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610c00.
02000000:00000001:2.0:1787764003.901853:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764003.901854:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764003.901855:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc000.
00000100:00000001:2.0:1787764003.901856:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764003.901857:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764003.901859:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764003.901861:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87be1c0 x1874599832681600/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901865:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764003.901866:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764003.901867:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764003.901868:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764003.901869:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764003.901869:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764003.901870:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764003.901871:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610800.
02000000:00000001:2.0:1787764003.901872:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764003.901873:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764003.901874:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87be1c0.
00000100:00000001:2.0:1787764003.901875:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764003.901876:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764003.901877:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764003.901879:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bed00 x1874599832681728/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764028 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764003.901882:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764003.901883:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764003.901884:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764003.901885:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764003.901886:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764003.901887:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764003.901888:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764003.901888:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612000.
02000000:00000001:2.0:1787764003.901890:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764003.901891:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764003.901892:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bed00.
00000100:00000001:2.0:1787764003.901893:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764003.901894:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764003.901896:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764003.901898:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764003.901899:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764003.901901:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764003.901902:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764004.540254:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764004.540256:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764004.540264:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764004.540264:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:1.0:1787764004.540269:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764004.540269:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764004.540274:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764004.540278:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764004.540279:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764004.540282:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764004.540285:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764004.540286:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764004.540290:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764004.540292:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764004.540295:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764004.540297:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764004.540298:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764004.540300:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764004.540303:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764004.540303:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764004.540308:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764004.540354:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764004.540357:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764004.540360:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764004.924256:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764004.924264:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764004.924267:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764004.924272:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764004.924277:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764004.924281:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764004.924284:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764004.924286:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764004.924290:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764005.180204:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316392179, at 4316392192
00000100:00000010:3.0:1787764005.180221:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7ea0.
00000100:00000040:2.0:1787764005.180242:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316392179, at 4316392192
00000100:00000010:2.0:1787764005.180250:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00010000:00000001:3.0:1787764005.564223:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764005.564228:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764005.564229:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764005.564234:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764005.564235:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764005.564237:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764005.564239:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764005.564241:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764005.564244:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764005.564245:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764005.564249:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764005.564252:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764005.564253:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764005.564255:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764005.564258:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764005.564288:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764005.564292:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764005.564295:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764005.948251:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764005.948259:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764005.948262:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764005.948268:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764005.948274:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764005.948277:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764005.948280:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764005.948283:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764005.948286:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764006.588248:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764006.588252:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764006.588256:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764006.588256:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764006.588259:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764006.588263:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764006.588264:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764006.588267:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:1.0:1787764006.588268:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764006.588270:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:1.0:1787764006.588273:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764006.588274:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764006.588276:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764006.588278:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:1.0:1787764006.588279:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764006.588281:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764006.588283:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764006.588284:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764006.588288:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764006.588291:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764006.588292:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764006.588341:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764006.588344:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764006.588347:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764006.972240:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764006.972247:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764006.972248:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764006.972253:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764006.972257:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764006.972259:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764006.972261:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764006.972264:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764006.972266:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764007.612221:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764007.612227:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764007.612229:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764007.612231:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764007.612234:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764007.612237:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764007.612238:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764007.612242:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764007.612244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764007.612246:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764007.612247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764007.612249:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764007.612252:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764007.612253:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764007.612254:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764007.612299:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764007.612303:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764007.612306:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764007.996260:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764007.996267:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764007.996269:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764007.996274:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764007.996278:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764007.996282:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764007.996284:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764007.996287:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764007.996290:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764008.636303:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764008.636305:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764008.636315:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764008.636317:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764008.636325:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764008.636327:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764008.636332:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764008.636334:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764008.636339:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764008.636344:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764008.636346:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764008.636352:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764008.636359:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764008.636363:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764008.636367:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764008.636442:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764008.636449:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764008.636455:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764009.020205:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764009.020211:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764009.020213:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764009.020218:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764009.020221:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764009.020225:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.020228:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764009.020231:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.020233:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764009.020235:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764009.020241:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787764009.020244:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764009.020247:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764009.020251:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764009.020258:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21723
00000400:00000200:1.0:1787764009.020265:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764009.020271:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764009.020274:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764009.020277:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764009.020279:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764009.020282:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764009.020284:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764009.020287:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764009.020288:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787764009.020290:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764009.020293:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787764009.020296:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787764009.020297:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787764009.020300:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764009.020301:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764009.020303:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764009.020304:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764009.020307:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787764009.020310:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764009.020311:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764009.020314:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764009.020316:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764009.020318:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764009.020320:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764009.020321:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764009.020329:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932940.
00000020:00000040:1.0:1787764009.020331:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787764009.020334:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764009.020336:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764009.020338:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764009.020340:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764009.020343:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764009.020352:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb000.
00000100:00000001:1.0:1787764009.020357:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764009.020365:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932940 x1874599832681856/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764009.020372:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764009.020375:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764009.020378:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000af90eaa] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932940 x1874599832681856/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764009.020394:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764009.020396:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764009.020398:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787764009.020399:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764009.020400:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764009.020402:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764009.020404:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21723
00000400:00000200:1.0:1787764009.020407:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764009.020409:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764009.020411:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764009.020413:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764009.020415:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764009.020415:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:1.0:1787764009.020419:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764009.020420:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764009.020421:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764009.020424:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1787764009.020425:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764009.020429:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787764009.020430:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764009.020432:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787764009.020434:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:2.0:1787764009.020435:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932940 x1874599832681856/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1787764009.020438:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787764009.020445:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764009.020449:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764009.020453:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:1.0:1787764009.020454:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764009.020456:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764009.020457:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764009.020460:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:2.0:1787764009.020460:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:1.0:1787764009.020463:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:2.0:1787764009.020464:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764009.020467:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764009.020468:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764009.020469:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:2.0:1787764009.020475:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832681856:192.168.203.110@tcp:250:kworker.0
02000000:00000001:1.0:1787764009.020476:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764009.020478:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764009.020479:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764009.020484:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1787764009.020484:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:1.0:1787764009.020488:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764009.020488:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1787764009.020492:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764009.020493:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764009.020495:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930780.
02000000:00000001:2.0:1787764009.020496:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:1.0:1787764009.020499:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764009.020502:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764009.020503:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764009.020504:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
00000100:00000001:1.0:1787764009.020505:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764009.020507:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764009.020509:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764009.020510:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764009.020515:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9c00.
00000400:00000200:2.0:1787764009.020516:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764009.020520:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764009.020523:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000100:00100000:1.0:1787764009.020527:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930780 x1874599832681984/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764009.020533:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764009.020535:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764009.020536:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000040:1.0:1787764009.020539:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3977950] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930780 x1874599832681984/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764009.020544:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832681856, portal 25
00000100:00000001:1.0:1787764009.020547:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764009.020548:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764009.020550:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497731904 : -115097211819712 : ffff9751d7932940)
00000100:00080000:1.0:1787764009.020553:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764009.020556:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787764009.020559:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:2.0:1787764009.020560:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932940 x1874599832681856/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764034 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787764009.020561:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764009.020566:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764009.020571:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21723
00000100:00000001:2.0:1787764009.020576:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:1.0:1787764009.020578:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:2.0:1787764009.020582:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764009.020585:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000010:2.0:1787764009.020587:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00080000:1.0:1787764009.020588:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000200:2.0:1787764009.020592:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832681856, offset 0
00000100:00000040:1.0:1787764009.020593:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764009.020597:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764009.020600:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764009.020604:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787764009.020605:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764009.020609:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764009.020610:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787764009.020611:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764009.020612:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000400:00000200:2.0:1787764009.020615:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:1.0:1787764009.020616:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787764009.020620:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787764009.020624:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764009.020625:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:1.0:1787764009.020626:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764009.020632:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764009.020633:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1787764009.020634:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764009.020638:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:2.0:1787764009.020640:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00080000:1.0:1787764009.020644:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:2.0:1787764009.020647:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1787764009.020649:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764009.020651:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1787764009.020652:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
02000000:00000001:1.0:1787764009.020655:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764009.020659:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764009.020659:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58562:58562:256:4294967295] 192.168.203.110@tcp LPNI seq info [58562:58562:8:4294967295]
00000100:00000001:1.0:1787764009.020664:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764009.020668:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:2.0:1787764009.020669:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000008:00000001:1.0:1787764009.020672:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764009.020675:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764009.020681:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932580.
00000400:00000200:2.0:1787764009.020682:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000020:00000040:1.0:1787764009.020686:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764009.020691:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:2.0:1787764009.020691:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
02000000:00000001:1.0:1787764009.020695:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764009.020697:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764009.020698:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000010:2.0:1787764009.020699:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
02000000:00000001:1.0:1787764009.020700:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764009.020706:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cac00.
00000800:00000200:2.0:1787764009.020708:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787764009.020709:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787764009.020713:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000100:00100000:1.0:1787764009.020717:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932580 x1874599832682112/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.1:1787764009.020722:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764009.020729:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:1.0:1787764009.020730:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764009.020734:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:2.0:1787764009.020739:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000040:1.0:1787764009.020742:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d026924] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932580 x1874599832682112/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:2.0:1787764009.020743:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787764009.020751:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:1.0:1787764009.020752:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764009.020756:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787764009.020760:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:1.0:1787764009.020764:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1787764009.020766:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000100:00080000:1.0:1787764009.020770:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:2.0:1787764009.020770:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787764009.020775:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:1.0:1787764009.020777:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787764009.020781:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000200:2.0:1787764009.020781:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932940 x1874599832681856/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764034 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787764009.020785:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764009.020792:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787764009.020798:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00080000:1.0:1787764009.020799:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21723
00000100:00000040:2.0:1787764009.020804:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932940 x1874599832681856/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764034 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787764009.020810:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764009.020817:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787764009.020819:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764009.020821:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787764009.020822:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:1.0:1787764009.020827:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764009.020829:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764009.020829:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764009.020834:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764009.020838:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764009.020839:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764009.020843:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316397299
00000020:00000001:1.0:1787764009.020844:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764009.020846:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787764009.020849:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000010:2.0:1787764009.020850:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000020:00000040:1.0:1787764009.020851:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787764009.020856:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787764009.020858:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:1.0:1787764009.020860:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764009.020860:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787764009.020865:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764009.020866:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764009.020868:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764009.020869:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316397299
00000020:00000040:1.0:1787764009.020871:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000010:2.0:1787764009.020874:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7b40.
00000100:00080000:1.0:1787764009.020875:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764009.020878:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787764009.020879:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764009.020883:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787764009.020883:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764009.020886:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764009.020886:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764009.020892:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764009.020893:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:1.0:1787764009.020897:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764009.020901:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00100000:2.0:1787764009.020901:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764009/real 1787764009]  req@ffff9751d7932940 x1874599832681856/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:1.0:1787764009.020906:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764009.020907:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764009.020910:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933840.
00000020:00000040:1.0:1787764009.020914:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787764009.020919:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764009.020921:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764009.020925:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000001:2.0:1787764009.020926:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:1.0:1787764009.020927:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000001:2.0:1787764009.020929:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:1.0:1787764009.020930:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764009.020932:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
02000000:00000010:1.0:1787764009.020935:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9400.
00000400:00000200:2.0:1787764009.020936:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000100:00000001:1.0:1787764009.020941:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764009.020941:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787764009.020943:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00100000:1.0:1787764009.020948:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933840 x1874599832682240/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:2.0:1787764009.020950:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932940 x1874599832681856/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764009.020975:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764009.020979:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764009.020979:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932940 x1874599832681856/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787764009.020988:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000083dd6d32] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933840 x1874599832682240/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021020:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764009.021022:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.021023:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764009.021024:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764009.021028:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764009.021030:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932940 x1874599832681856/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787764009.021033:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21733)
00000100:00000001:2.0:1787764009.021039:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764009.021043:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932940 x1874599832681856/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021048:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764009.021050:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.021051:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764009.021052:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.021054:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764009.021055:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764009.021057:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764009.021060:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764009.021061:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764009.021063:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932940 x1874599832681856/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764009.021069:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832681856:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764009.021071:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.021073:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764009.021074:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932940 x1874599832681856/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021078:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764009.021080:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764009.021083:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764009.021084:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764009.021086:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764009.021087:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764009.021088:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764009.021090:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb000.
02000000:00000001:2.0:1787764009.021093:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764009.021095:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764009.021096:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932940.
00000100:00000001:2.0:1787764009.021099:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764009.021099:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.021101:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.021106:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764009.021107:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764009.021108:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764009.021109:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764009.021111:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764009.021113:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930780 x1874599832681984/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021118:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764009.021119:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764009.021120:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764009.021122:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764009.021123:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764009.021125:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832681984:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764009.021127:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764009.021129:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764009.021130:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764009.021131:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764009.021150:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764009.021152:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764009.021155:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764009.021156:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787764009.021159:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787764009.021161:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832681984, portal 10
00000100:00000001:2.0:1787764009.021163:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764009.021175:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497723264 : -115097211828352 : ffff9751d7930780)
00000100:00000040:2.0:1787764009.021181:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930780 x1874599832681984/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764034 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021185:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764009.021187:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764009.021188:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787764009.021190:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832681984, offset 0
00000400:00000200:2.0:1787764009.021193:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764009.021196:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764009.021198:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764009.021200:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764009.021202:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764009.021204:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764009.021206:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764009.021208:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58563:58563:256:4294967295] 192.168.203.110@tcp LPNI seq info [58563:58563:8:4294967295]
00000400:00000200:2.0:1787764009.021211:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764009.021216:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764009.021219:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764009.021221:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787764009.021224:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764009.021226:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787764009.021229:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764009.021231:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1787764009.021233:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764009.021235:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787764009.021237:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764009.021239:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764009.021241:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787764009.021242:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787764009.021244:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764009.021246:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930780 x1874599832681984/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764034 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021256:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764009.021258:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930780 x1874599832681984/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764034 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021263:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764009.021264:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764009.021266:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764009.021268:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764009.021270:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764009.021272:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316396064
00000100:00000001:2.0:1787764009.021274:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316397299
00000100:00000001:2.0:1787764009.021275:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316396064
00000100:00000001:2.0:1787764009.021277:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316397299
00000100:00000001:2.0:1787764009.021278:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764009.021280:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764009.021281:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764009.021283:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764009/real 1787764009]  req@ffff9751d7930780 x1874599832681984/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764009.021293:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764009.021294:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764009.021295:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787764009.021297:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787764009.021298:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787764009.021300:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764009.021302:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930780 x1874599832681984/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764009.021307:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930780 x1874599832681984/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021311:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764009.021312:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.021313:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764009.021314:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764009.021317:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930780 x1874599832681984/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021321:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764009.021323:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930780 x1874599832681984/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021328:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764009.021329:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.021330:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764009.021331:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.021332:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764009.021341:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764009.021343:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764009.021345:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764009.021347:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764009.021349:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930780 x1874599832681984/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764009.021354:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832681984:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764009.021356:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764009.021358:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932580 x1874599832682112/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021362:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764009.021363:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764009.021364:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764009.021365:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764009.021367:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764009.021369:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832682112:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764009.021371:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764009.021372:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764009.021373:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764009.021375:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764009.021377:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764009.021378:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764009.021380:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764009.021382:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787764009.021384:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787764009.021386:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832682112, portal 10
00000100:00000001:2.0:1787764009.021387:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764009.021388:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497730944 : -115097211820672 : ffff9751d7932580)
00000100:00000040:2.0:1787764009.021392:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932580 x1874599832682112/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764034 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021396:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764009.021397:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764009.021398:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787764009.021400:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832682112, offset 0
00000400:00000200:2.0:1787764009.021402:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764009.021404:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764009.021406:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764009.021408:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764009.021410:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764009.021411:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764009.021413:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764009.021415:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58564:58564:256:4294967295] 192.168.203.110@tcp LPNI seq info [58564:58564:8:4294967295]
00000400:00000200:2.0:1787764009.021418:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764009.021422:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764009.021425:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764009.021426:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787764009.021429:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764009.021431:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787764009.021433:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764009.021435:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1787764009.021437:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764009.021439:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787764009.021441:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764009.021443:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764009.021445:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787764009.021446:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787764009.021448:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764009.021450:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932580 x1874599832682112/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764034 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021455:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764009.021457:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932580 x1874599832682112/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764034 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021461:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764009.021462:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764009.021464:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764009.021465:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764009.021467:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764009.021468:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316396052
00000100:00000001:2.0:1787764009.021470:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316397299
00000100:00000001:2.0:1787764009.021471:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316396052
00000100:00000001:2.0:1787764009.021473:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316397299
00000100:00000001:2.0:1787764009.021475:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764009.021476:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764009.021477:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764009.021479:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764009/real 1787764009]  req@ffff9751d7932580 x1874599832682112/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764009.021484:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764009.021485:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764009.021486:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787764009.021487:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787764009.021488:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787764009.021490:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764009.021492:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932580 x1874599832682112/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764009.021497:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932580 x1874599832682112/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021500:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764009.021501:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.021502:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764009.021503:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764009.021505:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932580 x1874599832682112/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021509:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764009.021511:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932580 x1874599832682112/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021515:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764009.021517:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.021518:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764009.021518:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.021520:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764009.021521:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764009.021522:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764009.021525:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764009.021526:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764009.021528:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932580 x1874599832682112/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764009.021537:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832682112:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764009.021540:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764009.021542:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933840 x1874599832682240/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021546:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764009.021546:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764009.021548:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764009.021549:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764009.021550:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764009.021551:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832682240:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764009.021553:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764009.021554:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764009.021555:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764009.021556:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764009.021558:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764009.021559:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764009.021561:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764009.021563:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787764009.021565:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787764009.021567:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832682240, portal 4
00000100:00000001:2.0:1787764009.021568:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764009.021569:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497735744 : -115097211815872 : ffff9751d7933840)
00000100:00000040:2.0:1787764009.021572:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933840 x1874599832682240/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764034 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021576:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764009.021578:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764009.021579:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787764009.021581:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832682240, offset 0
00000400:00000200:2.0:1787764009.021583:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764009.021585:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764009.021588:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764009.021590:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764009.021591:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764009.021593:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764009.021595:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764009.021597:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58565:58565:256:4294967295] 192.168.203.110@tcp LPNI seq info [58565:58565:8:4294967295]
00000400:00000200:2.0:1787764009.021599:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764009.021603:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764009.021606:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764009.021607:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787764009.021610:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764009.021611:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787764009.021614:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764009.021615:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1787764009.021618:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764009.021619:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787764009.021621:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764009.021623:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764009.021625:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787764009.021626:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787764009.021627:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764009.021630:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933840 x1874599832682240/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764034 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021635:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764009.021636:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933840 x1874599832682240/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764034 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021641:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764009.021642:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764009.021643:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764009.021645:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764009.021647:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764009.021648:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316396069
00000100:00000001:2.0:1787764009.021650:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316397299
00000100:00000001:2.0:1787764009.021652:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316396069
00000100:00000001:2.0:1787764009.021653:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316397299
00000100:00000001:2.0:1787764009.021654:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764009.021655:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764009.021656:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764009.021659:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764009/real 1787764009]  req@ffff9751d7933840 x1874599832682240/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764009.021663:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764009.021664:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764009.021665:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787764009.021666:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787764009.021667:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787764009.021668:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764009.021670:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933840 x1874599832682240/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764009.021675:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933840 x1874599832682240/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021679:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764009.021680:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.021681:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764009.021682:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764009.021684:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933840 x1874599832682240/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021688:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764009.021690:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933840 x1874599832682240/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021694:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764009.021696:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.021697:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764009.021698:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.021699:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764009.021700:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764009.021701:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764009.021703:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764009.021704:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764009.021707:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933840 x1874599832682240/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764009.021711:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832682240:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764009.021713:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.021714:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764009.021716:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930780 x1874599832681984/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021720:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764009.021721:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764009.021722:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764009.021724:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764009.021724:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764009.021725:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764009.021726:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764009.021727:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:2.0:1787764009.021729:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764009.021730:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764009.021731:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930780.
00000100:00000001:2.0:1787764009.021733:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764009.021734:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.021735:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764009.021736:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932580 x1874599832682112/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021745:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764009.021745:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764009.021746:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764009.021748:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764009.021749:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764009.021749:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764009.021750:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764009.021751:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cac00.
02000000:00000001:2.0:1787764009.021752:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764009.021753:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764009.021754:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932580.
00000100:00000001:2.0:1787764009.021755:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764009.021756:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.021757:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764009.021759:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933840 x1874599832682240/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764034 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764009.021762:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764009.021763:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764009.021764:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764009.021765:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764009.021766:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764009.021767:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764009.021768:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764009.021768:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9400.
02000000:00000001:2.0:1787764009.021769:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764009.021770:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764009.021771:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933840.
00000100:00000001:2.0:1787764009.021773:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764009.021774:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.021775:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764009.021777:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764009.021778:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764009.021779:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764009.021780:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764009.660211:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764009.660218:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764009.660219:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764009.660224:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764009.660227:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764009.660228:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764009.660233:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764009.660233:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764009.660236:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764009.660239:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764009.660241:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764009.660243:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764009.660245:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764009.660251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764009.660253:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764009.660313:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764009.660318:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764009.660323:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764010.044197:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764010.044204:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764010.044206:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764010.044211:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764010.044214:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764010.044216:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764010.044230:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764010.044233:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764010.044235:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764010.300314:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316397299, at 4316397312
00000100:00000010:3.0:1787764010.300336:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7b40.
00000100:00000040:2.0:1787764010.300364:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316397299, at 4316397312
00000100:00000010:2.0:1787764010.300381:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00010000:00000001:3.0:1787764010.684248:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764010.684254:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764010.684259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764010.684261:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764010.684265:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764010.684267:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764010.684276:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764010.684285:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764010.684288:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764010.684294:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764010.684300:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764010.684305:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764010.684310:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:2.0:1787764010.684311:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000100:00000001:1.0:1787764010.684315:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1787764010.684320:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764010.684321:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764010.684331:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764011.068259:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764011.068270:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764011.068273:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764011.068280:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764011.068286:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764011.068290:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764011.068294:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764011.068297:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764011.068301:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764011.708216:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764011.708222:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764011.708224:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764011.708226:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764011.708226:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764011.708231:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764011.708232:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764011.708233:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764011.708235:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764011.708237:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764011.708238:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764011.708240:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764011.708240:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:1.0:1787764011.708243:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764011.708243:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:1.0:1787764011.708246:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764011.708246:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764011.708250:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764011.708252:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764011.708254:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764011.708257:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764011.708315:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764011.708318:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764011.708321:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764012.092249:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764012.092258:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764012.092261:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764012.092267:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764012.092271:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764012.092276:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764012.092278:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764012.092282:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764012.092285:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764012.732271:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764012.732271:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764012.732279:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764012.732280:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764012.732283:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764012.732290:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764012.732293:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764012.732295:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764012.732297:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764012.732300:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764012.732304:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764012.732307:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764012.732307:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764012.732313:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764012.732313:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764012.732385:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764012.732389:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764012.732392:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764013.116218:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764013.116224:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764013.116226:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764013.116231:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764013.116234:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764013.116237:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764013.116240:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764013.116242:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764013.116245:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764013.756267:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764013.756275:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764013.756276:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764013.756288:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764013.756288:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764013.756295:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764013.756296:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764013.756299:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764013.756303:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764013.756307:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764013.756316:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764013.756321:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764013.756326:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764013.756331:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764013.756336:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764013.756374:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764013.756381:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764013.756387:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764014.140230:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764014.140239:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764014.140243:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764014.140244:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1787764014.140250:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:3.0:1787764014.140252:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764014.140255:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764014.140259:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764014.140263:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787764014.140266:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764014.140270:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1787764014.140272:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21728
00000100:00000001:2.0:1787764014.140276:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764014.140280:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:3.0:1787764014.140285:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764014.140286:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764014.140298:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764014.140301:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764014.140304:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764014.140307:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764014.140309:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764014.140311:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764014.140314:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764014.140316:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787764014.140318:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764014.140321:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787764014.140323:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787764014.140324:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787764014.140326:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764014.140328:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764014.140329:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764014.140331:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764014.140333:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787764014.140336:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764014.140337:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764014.140340:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764014.140342:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764014.140344:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764014.140345:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764014.140347:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764014.140355:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bde00.
00000020:00000040:3.0:1787764014.140364:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787764014.140367:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764014.140368:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764014.140371:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764014.140373:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764014.140376:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764014.140383:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610400.
00000100:00000001:3.0:1787764014.140389:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764014.140396:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bde00 x1874599832682368/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764014.140404:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764014.140408:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764014.140412:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000941163c2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bde00 x1874599832682368/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764014.140427:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764014.140430:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764014.140432:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787764014.140433:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764014.140435:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764014.140437:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764014.140439:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21728
00000400:00000200:3.0:1787764014.140441:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764014.140444:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764014.140446:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764014.140448:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764014.140450:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764014.140452:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764014.140453:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764014.140455:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:3.0:1787764014.140456:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764014.140458:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:3.0:1787764014.140460:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1787764014.140461:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000040:3.0:1787764014.140462:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000001:2.0:1787764014.140467:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1787764014.140467:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:3.0:1787764014.140472:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:3.0:1787764014.140477:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000040:2.0:1787764014.140479:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bde00 x1874599832682368/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764014.140480:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764014.140484:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764014.140488:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764014.140492:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:3.0:1787764014.140497:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764014.140501:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764014.140503:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:3.0:1787764014.140504:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787764014.140506:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764014.140509:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764014.140510:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764014.140512:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764014.140515:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764014.140516:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764014.140519:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:2.0:1787764014.140523:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832682368:192.168.203.110@tcp:250:kworker.0
00000008:00100000:3.0:1787764014.140524:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1787764014.140528:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:3.0:1787764014.140529:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764014.140532:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1787764014.140532:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764014.140535:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764014.140535:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bfc00.
02000000:00000001:2.0:1787764014.140541:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:3.0:1787764014.140541:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787764014.140545:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764014.140546:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764014.140548:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
00000100:00000001:3.0:1787764014.140548:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764014.140551:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764014.140553:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764014.140554:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764014.140560:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000010:3.0:1787764014.140560:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611800.
00000100:00000001:3.0:1787764014.140565:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764014.140566:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000100:00100000:3.0:1787764014.140571:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bfc00 x1874599832682496/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:2.0:1787764014.140573:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000001:3.0:1787764014.140580:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:2.0:1787764014.140583:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832682368, portal 25
00000100:00000001:3.0:1787764014.140583:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764014.140589:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000040:3.0:1787764014.140590:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003c1f6b31] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bfc00 x1874599832682496/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.140591:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049853440 : -115096659698176 : ffff9751f87bde00)
00000100:00000040:2.0:1787764014.140601:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bde00 x1874599832682368/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764039 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764014.140602:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764014.140607:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764014.140614:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1787764014.140617:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787764014.140619:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:3.0:1787764014.140619:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:2.0:1787764014.140625:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787764014.140626:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:2.0:1787764014.140628:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00080000:3.0:1787764014.140631:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21728
00000100:00000200:2.0:1787764014.140633:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832682368, offset 0
00000400:00000200:3.0:1787764014.140637:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764014.140639:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764014.140641:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764014.140642:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764014.140644:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764014.140646:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764014.140650:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787764014.140651:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764014.140654:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764014.140655:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:3.0:1787764014.140656:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764014.140657:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:3.0:1787764014.140660:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:2.0:1787764014.140661:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:3.0:1787764014.140664:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:3.0:1787764014.140668:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764014.140671:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:3.0:1787764014.140672:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764014.140678:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764014.140679:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1787764014.140684:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787764014.140686:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764014.140691:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000020:00000040:3.0:1787764014.140691:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:2.0:1787764014.140696:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00080000:3.0:1787764014.140700:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:2.0:1787764014.140701:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58566:58566:256:4294967295] 192.168.203.110@tcp LPNI seq info [58566:58566:8:4294967295]
00000400:00000200:2.0:1787764014.140708:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764014.140708:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764014.140713:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1787764014.140719:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
02000000:00000001:3.0:1787764014.140719:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764014.140725:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787764014.140726:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787764014.140729:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000010:2.0:1787764014.140732:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000008:00100000:3.0:1787764014.140733:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787764014.140736:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787764014.140740:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787764014.140740:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764014.140745:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc3c0.
00000800:00000200:2.0:1787764014.140746:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000020:00000040:3.0:1787764014.140751:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000800:00000200:2.1:1787764014.140756:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:3.0:1787764014.140757:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764014.140759:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764014.140762:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:2.1:1787764014.140762:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
02000000:00000001:3.0:1787764014.140765:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000100:2.0:1787764014.140768:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764014.140770:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787764014.140774:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:3.0:1787764014.140775:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764014.140780:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
02000000:00000010:3.0:1787764014.140781:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613c00.
00000400:00000200:2.0:1787764014.140784:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000100:00000001:3.0:1787764014.140785:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764014.140789:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00100000:3.0:1787764014.140794:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc3c0 x1874599832682624/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764014.140795:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764014.140802:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bde00 x1874599832682368/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764039 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764014.140804:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764014.140808:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764014.140816:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000060bf8470] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc3c0 x1874599832682624/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.140818:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764014.140824:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bde00 x1874599832682368/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764039 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764014.140833:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764014.140835:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764014.140838:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:3.0:1787764014.140838:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764014.140842:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764014.140844:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1787764014.140848:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764014.140853:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316402419
00000100:00080000:3.0:1787764014.140853:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000010:2.0:1787764014.140857:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00080000:3.0:1787764014.140859:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787764014.140864:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787764014.140870:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787764014.140871:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316402419
00000100:00080000:3.0:1787764014.140873:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000010:2.0:1787764014.140875:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7ae0.
00000100:00000001:3.0:1787764014.140877:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787764014.140879:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00080000:3.0:1787764014.140883:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21728
00000100:00000001:2.0:1787764014.140885:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764014.140887:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764014.140890:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764014.140892:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000400:00000200:3.0:1787764014.140896:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00100000:2.0:1787764014.140899:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764014/real 1787764014]  req@ffff9751f87bde00 x1874599832682368/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787764014.140900:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764014.140906:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764014.140911:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000001:2.0:1787764014.140915:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:3.0:1787764014.140916:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000001:2.0:1787764014.140919:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:3.0:1787764014.140920:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000010:2.0:1787764014.140922:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000020:00000001:3.0:1787764014.140923:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:2.0:1787764014.140926:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000020:00000040:3.0:1787764014.140926:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:3.0:1787764014.140928:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:2.0:1787764014.140930:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000020:00000040:3.0:1787764014.140931:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000001:2.0:1787764014.140934:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000001:3.0:1787764014.140935:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:3.0:1787764014.140938:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000200:2.0:1787764014.140940:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bde00 x1874599832682368/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787764014.140943:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764014.140948:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764014.140952:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764014.140955:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000200:2.0:1787764014.140956:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bde00 x1874599832682368/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787764014.140958:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:3.0:1787764014.140964:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764014.140970:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764014.140971:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764014.140974:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787764014.140974:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787764014.140978:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:3.0:1787764014.140978:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764014.140979:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787764014.140985:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764014.140987:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bde00 x1874599832682368/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764014.140990:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787764014.140995:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:2.0:1787764014.140996:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000008:00000001:3.0:1787764014.141002:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764014.141003:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bde00 x1874599832682368/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764014.141009:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764014.141014:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764014.141017:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:3.0:1787764014.141017:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc780.
00000100:00000001:2.0:1787764014.141018:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764014.141020:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764014.141024:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000020:00000040:3.0:1787764014.141025:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:2.0:1787764014.141026:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764014.141031:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000001:3.0:1787764014.141032:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764014.141035:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764014.141037:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764014.141038:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764014.141041:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787764014.141044:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
02000000:00000001:3.0:1787764014.141045:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764014.141051:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613800.
00000100:00000040:2.0:1787764014.141054:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bde00 x1874599832682368/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764014.141055:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764014.141063:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc780 x1874599832682752/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764014.141076:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00100000:2.0:1787764014.141077:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832682368:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764014.141078:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764014.141085:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7d56184] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc780 x1874599832682752/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.141087:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764014.141092:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1787764014.141097:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764014.141101:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bde00 x1874599832682368/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787764014.141103:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21738)
00000100:00000001:2.0:1787764014.141113:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764014.141117:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764014.141120:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764014.141124:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764014.141126:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764014.141163:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764014.141166:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764014.141169:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610400.
02000000:00000001:2.0:1787764014.141173:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764014.141175:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764014.141177:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bde00.
00000100:00000001:2.0:1787764014.141180:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764014.141181:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764014.141184:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764014.141190:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764014.141191:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764014.141194:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764014.141196:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764014.141197:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764014.141202:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bfc00 x1874599832682496/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.141210:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764014.141212:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764014.141214:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764014.141216:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764014.141219:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764014.141223:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832682496:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764014.141226:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764014.141229:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764014.141231:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764014.141233:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764014.141236:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764014.141239:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764014.141243:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764014.141246:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787764014.141250:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787764014.141253:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832682496, portal 10
00000100:00000001:2.0:1787764014.141256:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764014.141257:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049861120 : -115096659690496 : ffff9751f87bfc00)
00000100:00000040:2.0:1787764014.141265:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bfc00 x1874599832682496/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764039 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.141273:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764014.141276:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764014.141278:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787764014.141281:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832682496, offset 0
00000400:00000200:2.0:1787764014.141286:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764014.141290:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764014.141295:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764014.141298:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764014.141301:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764014.141304:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764014.141307:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764014.141310:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58567:58567:256:4294967295] 192.168.203.110@tcp LPNI seq info [58567:58567:8:4294967295]
00000400:00000200:2.0:1787764014.141315:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764014.141331:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764014.141337:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764014.141342:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787764014.141346:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764014.141350:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787764014.141355:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764014.141358:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1787764014.141362:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764014.141364:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787764014.141367:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764014.141371:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764014.141374:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787764014.141376:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787764014.141379:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764014.141384:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bfc00 x1874599832682496/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764039 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.141393:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764014.141397:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bfc00 x1874599832682496/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764039 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.141412:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764014.141414:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764014.141418:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764014.141422:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764014.141426:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764014.141428:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316401184
00000100:00000001:2.0:1787764014.141431:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316402419
00000100:00000001:2.0:1787764014.141434:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316401184
00000100:00000001:2.0:1787764014.141436:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316402419
00000100:00000001:2.0:1787764014.141439:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764014.141441:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764014.141443:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764014.141448:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764014/real 1787764014]  req@ffff9751f87bfc00 x1874599832682496/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764014.141457:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764014.141459:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764014.141461:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787764014.141464:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787764014.141465:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787764014.141468:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764014.141471:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bfc00 x1874599832682496/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764014.141482:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bfc00 x1874599832682496/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.141489:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764014.141491:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764014.141493:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764014.141494:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764014.141499:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bfc00 x1874599832682496/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.141507:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764014.141512:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bfc00 x1874599832682496/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.141520:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764014.141523:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764014.141525:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764014.141526:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764014.141528:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764014.141541:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764014.141544:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764014.141547:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764014.141550:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764014.141554:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bfc00 x1874599832682496/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764014.141564:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832682496:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764014.141568:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764014.141572:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc3c0 x1874599832682624/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.141579:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764014.141581:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764014.141583:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764014.141585:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764014.141588:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764014.141591:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832682624:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764014.141595:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764014.141597:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764014.141600:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764014.141601:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764014.141605:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764014.141608:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764014.141611:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764014.141614:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787764014.141617:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787764014.141620:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832682624, portal 10
00000100:00000001:2.0:1787764014.141623:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764014.141624:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049846720 : -115096659704896 : ffff9751f87bc3c0)
00000100:00000040:2.0:1787764014.141631:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc3c0 x1874599832682624/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764039 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.141639:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764014.141642:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764014.141644:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787764014.141647:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832682624, offset 0
00000400:00000200:2.0:1787764014.141651:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764014.141655:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764014.141660:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764014.141663:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764014.141666:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764014.141669:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764014.141671:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764014.141675:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58568:58568:256:4294967295] 192.168.203.110@tcp LPNI seq info [58568:58568:8:4294967295]
00000400:00000200:2.0:1787764014.141680:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764014.141687:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764014.141692:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764014.141695:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787764014.141700:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764014.141703:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787764014.141708:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764014.141711:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1787764014.141716:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764014.141718:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787764014.141721:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764014.141725:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764014.141728:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787764014.141730:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787764014.141733:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764014.141737:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc3c0 x1874599832682624/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764039 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.141746:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764014.141749:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc3c0 x1874599832682624/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764039 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.141757:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764014.141758:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764014.141761:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764014.141765:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764014.141768:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764014.141770:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316401172
00000100:00000001:2.0:1787764014.141773:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316402419
00000100:00000001:2.0:1787764014.141775:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316401172
00000100:00000001:2.0:1787764014.141778:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316402419
00000100:00000001:2.0:1787764014.141780:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764014.141782:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764014.141785:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764014.141789:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764014/real 1787764014]  req@ffff9751f87bc3c0 x1874599832682624/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764014.141798:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764014.141799:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764014.141801:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787764014.141804:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787764014.141805:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787764014.141808:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764014.141812:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc3c0 x1874599832682624/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764014.141822:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc3c0 x1874599832682624/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.141829:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764014.141831:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764014.141833:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764014.141835:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764014.141839:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc3c0 x1874599832682624/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.141847:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764014.141851:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc3c0 x1874599832682624/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.141859:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764014.141862:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764014.141863:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764014.141865:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764014.141867:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764014.141869:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764014.141871:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764014.141875:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764014.141877:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764014.141882:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc3c0 x1874599832682624/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764014.141891:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832682624:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764014.141896:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764014.141900:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc780 x1874599832682752/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.141907:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764014.141909:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764014.141912:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764014.141914:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764014.141916:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764014.141919:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832682752:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764014.141922:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764014.141924:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764014.141926:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764014.141928:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764014.141931:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764014.141933:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764014.141937:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764014.141940:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787764014.141944:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787764014.141947:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832682752, portal 4
00000100:00000001:2.0:1787764014.141949:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764014.141950:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049847680 : -115096659703936 : ffff9751f87bc780)
00000100:00000040:2.0:1787764014.141956:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc780 x1874599832682752/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764039 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.141964:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764014.141966:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764014.141969:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787764014.141972:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832682752, offset 0
00000400:00000200:2.0:1787764014.141976:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764014.141980:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764014.141984:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764014.141987:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764014.141990:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764014.141993:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764014.141995:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764014.141999:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58569:58569:256:4294967295] 192.168.203.110@tcp LPNI seq info [58569:58569:8:4294967295]
00000400:00000200:2.0:1787764014.142004:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764014.142011:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764014.142016:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764014.142020:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787764014.142024:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764014.142028:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787764014.142033:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764014.142036:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1787764014.142040:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764014.142042:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787764014.142046:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764014.142050:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764014.142053:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787764014.142055:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787764014.142058:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764014.142062:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc780 x1874599832682752/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764039 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.142071:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764014.142074:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc780 x1874599832682752/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764039 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.142082:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764014.142084:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764014.142087:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764014.142090:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764014.142094:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764014.142096:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316401189
00000100:00000001:2.0:1787764014.142099:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316402419
00000100:00000001:2.0:1787764014.142101:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316401189
00000100:00000001:2.0:1787764014.142103:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316402419
00000100:00000001:2.0:1787764014.142105:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764014.142108:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764014.142111:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764014.142115:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764014/real 1787764014]  req@ffff9751f87bc780 x1874599832682752/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764014.142124:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764014.142126:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764014.142128:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787764014.142165:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787764014.142167:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787764014.142170:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764014.142174:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc780 x1874599832682752/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764014.142184:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc780 x1874599832682752/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.142192:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764014.142194:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764014.142196:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764014.142198:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764014.142202:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc780 x1874599832682752/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.142210:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764014.142214:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc780 x1874599832682752/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.142222:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764014.142224:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764014.142226:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764014.142227:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764014.142229:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764014.142231:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764014.142234:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764014.142238:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764014.142240:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764014.142245:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc780 x1874599832682752/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764014.142254:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832682752:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764014.142259:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764014.142261:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764014.142264:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bfc00 x1874599832682496/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.142272:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764014.142273:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764014.142275:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764014.142278:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764014.142280:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764014.142281:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764014.142284:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764014.142286:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611800.
02000000:00000001:2.0:1787764014.142289:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764014.142290:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764014.142292:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bfc00.
00000100:00000001:2.0:1787764014.142294:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764014.142296:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764014.142297:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764014.142301:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc3c0 x1874599832682624/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.142309:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764014.142310:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764014.142312:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764014.142314:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764014.142316:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764014.142317:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764014.142319:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764014.142321:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613c00.
02000000:00000001:2.0:1787764014.142323:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764014.142324:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764014.142326:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc3c0.
00000100:00000001:2.0:1787764014.142328:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764014.142330:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764014.142331:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764014.142334:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc780 x1874599832682752/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764039 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764014.142341:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764014.142343:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764014.142344:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764014.142346:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764014.142348:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764014.142349:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764014.142351:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764014.142353:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613800.
02000000:00000001:2.0:1787764014.142355:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764014.142356:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764014.142357:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc780.
00000100:00000001:2.0:1787764014.142359:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764014.142360:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764014.142362:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764014.142366:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764014.142367:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764014.142369:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764014.142371:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764014.780273:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764014.780297:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764014.780300:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764014.780305:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764014.780309:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764014.780312:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764014.780315:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764014.780318:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764014.780318:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764014.780321:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764014.780333:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764014.780338:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764014.780355:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764014.780361:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764014.780364:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764014.780367:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764014.780370:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764014.780376:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764014.780378:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764014.780383:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764014.780384:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764014.780478:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764014.780483:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764014.780487:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764015.164210:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764015.164217:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764015.164219:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764015.164224:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764015.164227:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764015.164230:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764015.164232:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764015.164234:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764015.164237:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764015.420228:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316402419, at 4316402432
00000100:00000010:3.0:1787764015.420246:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7ae0.
00000100:00000040:2.0:1787764015.420272:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316402419, at 4316402432
00000100:00000010:2.0:1787764015.420281:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00000100:00000001:1.0:1787764015.804217:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764015.804220:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764015.804225:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764015.804226:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764015.804229:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764015.804233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764015.804234:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764015.804237:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764015.804239:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764015.804243:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764015.804245:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764015.804248:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764015.804254:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764015.804257:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764015.804262:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764015.804315:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764015.804321:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764015.804325:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764016.188235:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764016.188248:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764016.188250:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764016.188255:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764016.188258:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764016.188262:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764016.188264:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764016.188266:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764016.188268:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764016.828229:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764016.828230:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764016.828238:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764016.828240:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764016.828243:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764016.828247:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764016.828251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764016.828254:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764016.828258:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:1.0:1787764016.828259:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764016.828263:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:1.0:1787764016.828264:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764016.828270:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764016.828270:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764016.828276:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764016.828277:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764016.828282:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764016.828285:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764016.828287:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764016.828292:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764016.828293:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764016.828334:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764016.828337:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764016.828340:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764017.212245:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764017.212253:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764017.212256:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764017.212262:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764017.212267:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764017.212270:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764017.212273:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764017.212277:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764017.212280:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764017.852241:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764017.852247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764017.852249:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764017.852252:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764017.852253:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764017.852259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764017.852260:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764017.852261:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764017.852268:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764017.852268:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764017.852271:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764017.852274:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764017.852278:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764017.852281:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764017.852285:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764017.852368:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764017.852374:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764017.852378:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764018.236257:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764018.236263:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764018.236265:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764018.236269:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764018.236290:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764018.236293:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764018.236303:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764018.236306:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764018.236308:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764018.876232:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764018.876239:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764018.876242:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764018.876243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764018.876248:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764018.876249:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764018.876263:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764018.876264:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764018.876268:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764018.876269:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764018.876274:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764018.876276:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764018.876283:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764018.876283:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764018.876292:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764018.876350:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764018.876355:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764018.876358:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764019.260208:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764019.260215:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764019.260216:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764019.260222:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764019.260227:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764019.260230:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.260235:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764019.260243:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.260247:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764019.260248:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764019.260257:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764019.260259:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764019.260270:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764019.260277:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764019.260289:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21733
00000400:00000200:3.0:1787764019.260300:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764019.260309:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764019.260314:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764019.260320:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764019.260323:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764019.260325:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764019.260328:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764019.260332:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764019.260334:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787764019.260337:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764019.260342:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787764019.260344:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787764019.260348:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787764019.260351:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764019.260355:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764019.260357:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764019.260359:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764019.260361:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787764019.260364:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764019.260366:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764019.260369:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764019.260372:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764019.260375:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764019.260377:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764019.260380:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764019.260387:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bcf00.
00000020:00000040:3.0:1787764019.260391:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787764019.260394:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764019.260396:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764019.260398:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764019.260401:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764019.260405:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764019.260413:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612400.
00000100:00000001:3.0:1787764019.260422:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764019.260432:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bcf00 x1874599832682880/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764019.260442:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764019.260446:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764019.260451:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ebc1bc5] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bcf00 x1874599832682880/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764019.260477:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764019.260482:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:3.0:1787764019.260483:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764019.260488:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1787764019.260490:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787764019.260492:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1787764019.260496:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764019.260501:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:2.0:1787764019.260508:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bcf00 x1874599832682880/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764019.260511:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764019.260523:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21733
00000100:00000001:2.0:1787764019.260530:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764019.260534:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764019.260535:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:2.0:1787764019.260544:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000400:00000200:3.0:1787764019.260545:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
02000000:00000001:2.0:1787764019.260550:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:3.0:1787764019.260551:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
02000000:00000001:2.0:1787764019.260559:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764019.260559:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764019.260568:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:2.0:1787764019.260570:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832682880:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764019.260575:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764019.260579:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764019.260582:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764019.260584:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764019.260596:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764019.260600:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764019.260606:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764019.260609:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787764019.260616:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787764019.260622:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832682880, portal 25
00000100:00000001:2.0:1787764019.260624:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764019.260626:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049849600 : -115096659702016 : ffff9751f87bcf00)
00000100:00000040:2.0:1787764019.260632:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bcf00 x1874599832682880/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.260638:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764019.260645:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764019.260651:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000040:3.0:1787764019.260657:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000200:2.0:1787764019.260658:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832682880, offset 0
00000100:00000001:3.0:1787764019.260667:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787764019.260677:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000020:00000001:3.0:1787764019.260677:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764019.260682:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000400:00000200:2.0:1787764019.260693:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:3.0:1787764019.260695:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764019.260701:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:3.0:1787764019.260705:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000400:00000200:2.0:1787764019.260709:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764019.260716:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764019.260721:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764019.260725:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764019.260728:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764019.260731:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58570:58570:256:4294967295] 192.168.203.110@tcp LPNI seq info [58570:58570:8:4294967295]
00000400:00000200:2.0:1787764019.260735:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764019.260743:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764019.260748:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764019.260753:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787764019.260759:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764019.260762:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787764019.260766:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764019.260769:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1787764019.260772:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764019.260774:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787764019.260778:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764019.260781:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764019.260785:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787764019.260787:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787764019.260791:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764019.260795:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bcf00 x1874599832682880/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.260803:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764019.260806:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bcf00 x1874599832682880/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764044 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.260811:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764019.260813:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764019.260818:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764019.260821:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764019.260824:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316407539
00000100:00000010:2.0:1787764019.260827:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00000001:2.0:1787764019.260833:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787764019.260840:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316407539
00000100:00000010:2.0:1787764019.260842:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb77e0.
00000100:00000001:2.0:1787764019.260844:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787764019.260848:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764019.260850:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764019.260858:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000020:00000001:3.0:1787764019.260858:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00100000:2.0:1787764019.260866:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764019/real 1787764019]  req@ffff9751f87bcf00 x1874599832682880/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787764019.260873:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000001:2.0:1787764019.260876:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764019.260879:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:3.0:1787764019.260881:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1787764019.260882:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787764019.260890:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787764019.260893:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787764019.260897:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764019.260901:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bcf00 x1874599832682880/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764019.260907:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bcf00 x1874599832682880/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.260912:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764019.260913:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.260915:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764019.260917:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764019.260920:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bcf00 x1874599832682880/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.260924:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764019.260927:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bcf00 x1874599832682880/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.260933:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764019.260935:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.260936:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764019.260938:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764019.260939:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764019.260941:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764019.260946:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764019.260946:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764019.260956:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:2.0:1787764019.260957:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764019.260967:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00080000:2.0:1787764019.260968:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764019.260979:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764019.260980:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
02000000:00000001:3.0:1787764019.260993:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:2.0:1787764019.260998:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bcf00 x1874599832682880/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787764019.261004:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:2.0:1787764019.261009:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832682880:192.168.203.110@tcp:250:kworker.0
02000000:00000001:3.0:1787764019.261012:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764019.261018:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764019.261019:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.261026:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000008:00100000:3.0:1787764019.261029:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000040:2.0:1787764019.261030:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bcf00 x1874599832682880/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:3.0:1787764019.261037:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764019.261040:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764019.261044:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc780.
00000020:00000040:3.0:1787764019.261051:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000100:00000001:2.0:1787764019.261060:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764019.261061:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764019.261065:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764019.261066:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000001:3.0:1787764019.261071:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000010:2.0:1787764019.261072:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:3.0:1787764019.261074:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764019.261083:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
02000000:00000001:3.0:1787764019.261083:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764019.261089:0:257575:0:(genops.c:1217:class_import_put()) Process entered
02000000:00000010:3.0:1787764019.261092:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612c00.
00000020:00000040:2.0:1787764019.261104:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764019.261112:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764019.261116:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612400.
00000100:00000001:3.0:1787764019.261116:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764019.261126:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764019.261128:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764019.261165:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bcf00.
00000100:00000001:2.0:1787764019.261169:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00100000:3.0:1787764019.261169:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc780 x1874599832683008/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764019.261175:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.261188:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764019.261197:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764019.261200:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764019.261202:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764019.261204:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764019.261209:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7d56184] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc780 x1874599832683008/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.261211:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764019.261215:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764019.261233:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764019.261236:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764019.261239:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1787764019.261241:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764019.261243:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764019.261245:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764019.261246:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787764019.261251:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1787764019.261255:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21733
00000100:00000001:2.0:1787764019.261256:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764019.261268:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc780 x1874599832683008/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764019.261269:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764019.261283:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764019.261289:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787764019.261292:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764019.261295:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764019.261297:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764019.261303:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1787764019.261304:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1787764019.261309:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1787764019.261310:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764019.261317:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764019.261318:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764019.261324:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764019.261327:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000100:00100000:2.0:1787764019.261329:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832683008:192.168.203.110@tcp:38:kworker.0
00000020:00000001:3.0:1787764019.261331:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764019.261337:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000100:00000001:2.0:1787764019.261338:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:3.0:1787764019.261343:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
02000000:00000001:2.0:1787764019.261345:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000001:3.0:1787764019.261352:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
02000000:00000001:2.0:1787764019.261353:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764019.261364:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:3.0:1787764019.261370:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000010:2.0:1787764019.261372:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
00000100:00000001:3.0:1787764019.261377:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1787764019.261378:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764019.261383:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764019.261390:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764019.261392:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000010:2.0:1787764019.261395:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000020:00000040:3.0:1787764019.261396:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000010:2.0:1787764019.261401:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00080000:3.0:1787764019.261404:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000200:2.0:1787764019.261407:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832683008, portal 10
00000100:00000001:3.0:1787764019.261408:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764019.261416:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:3.0:1787764019.261416:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787764019.261425:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049847680 : -115096659703936 : ffff9751f87bc780)
02000000:00000001:3.0:1787764019.261427:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764019.261436:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764019.261439:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:2.0:1787764019.261441:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc780 x1874599832683008/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:3.0:1787764019.261442:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787764019.261451:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764019.261461:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764019.261462:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764019.261473:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000010:3.0:1787764019.261474:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc3c0.
00000400:00000010:2.0:1787764019.261481:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000020:00000040:3.0:1787764019.261481:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787764019.261496:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000200:2.0:1787764019.261497:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832683008, offset 0
02000000:00000001:3.0:1787764019.261504:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764019.261511:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:2.0:1787764019.261512:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
02000000:00000001:3.0:1787764019.261513:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764019.261517:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764019.261522:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000010:3.0:1787764019.261522:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612800.
00000100:00000001:3.0:1787764019.261528:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764019.261536:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc3c0 x1874599832683136/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787764019.261539:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764019.261555:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764019.261559:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764019.261566:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1787764019.261570:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1787764019.261571:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:3.0:1787764019.261579:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764019.261583:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58571:58571:256:4294967295] 192.168.203.110@tcp LPNI seq info [58571:58571:8:4294967295]
00000100:00000040:3.0:1787764019.261590:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000060bf8470] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc3c0 x1874599832683136/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764019.261598:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764019.261616:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764019.261618:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:3.0:1787764019.261627:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:2.0:1787764019.261628:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764019.261637:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000100:00000040:3.0:1787764019.261639:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:2.0:1787764019.261647:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764019.261651:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000100:00080000:3.0:1787764019.261651:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764019.261664:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000800:00000200:2.1:1787764019.261664:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787764019.261674:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:2.1:1787764019.261676:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000100:00080000:3.0:1787764019.261677:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764019.261682:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764019.261688:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21733
00000400:00000200:3.0:1787764019.261697:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000100:2.0:1787764019.261698:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764019.261704:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000800:00000010:2.0:1787764019.261706:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000100:00080000:3.0:1787764019.261707:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764019.261717:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764019.261718:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1787764019.261720:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764019.261729:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764019.261736:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1787764019.261737:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764019.261747:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:2.0:1787764019.261750:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000020:00000040:3.0:1787764019.261751:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:3.0:1787764019.261764:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:2.0:1787764019.261765:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000020:00000040:3.0:1787764019.261772:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000001:2.0:1787764019.261774:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000020:00000001:3.0:1787764019.261778:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000200:2.0:1787764019.261785:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc780 x1874599832683008/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787764019.261785:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:3.0:1787764019.261798:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764019.261807:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764019.261814:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764019.261817:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1787764019.261818:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:2.0:1787764019.261825:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc780 x1874599832683008/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764044 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787764019.261826:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:3.0:1787764019.261838:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764019.261849:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764019.261859:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787764019.261860:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764019.261866:0:257575:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:3.0:1787764019.261867:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764019.261870:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764019.261871:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764019.261884:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:2.0:1787764019.261886:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000008:00100000:3.0:1787764019.261889:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1787764019.261897:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764019.261899:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764019.261905:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764019.261906:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316406304
00000100:00000001:2.0:1787764019.261912:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316407539
00000100:00000010:3.0:1787764019.261912:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bfc00.
00000020:00000040:3.0:1787764019.261919:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:2.0:1787764019.261921:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316406304
00000100:00000001:2.0:1787764019.261932:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316407539
02000000:00000001:3.0:1787764019.261932:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764019.261939:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764019.261942:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764019.261943:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1787764019.261952:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764019.261952:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787764019.261960:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000001:3.0:1787764019.261960:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764019.261970:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613000.
00000100:00100000:2.0:1787764019.261971:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764019/real 1787764019]  req@ffff9751f87bc780 x1874599832683008/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764019.261977:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:2.0:1787764019.261988:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764019.261990:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00100000:3.0:1787764019.261991:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bfc00 x1874599832683264/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:2.0:1787764019.261992:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787764019.262000:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787764019.262007:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787764019.262015:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:3.0:1787764019.262019:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:2.0:1787764019.262024:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc780 x1874599832683008/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764019.262025:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764019.262040:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003c1f6b31] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bfc00 x1874599832683264/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764019.262054:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc780 x1874599832683008/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764019.262068:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764019.262080:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000040:3.0:1787764019.262084:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21743)
00000100:00000001:2.0:1787764019.262085:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.262106:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764019.262107:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764019.262113:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc780 x1874599832683008/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.262119:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764019.262123:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc780 x1874599832683008/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.262128:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764019.262130:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.262131:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764019.262132:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.262151:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764019.262170:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764019.262174:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764019.262178:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764019.262180:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764019.262182:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc780 x1874599832683008/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764019.262188:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832683008:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764019.262191:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.262193:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764019.262195:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc780 x1874599832683008/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.262199:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764019.262200:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764019.262201:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764019.262203:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764019.262205:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764019.262206:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764019.262207:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764019.262209:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612c00.
02000000:00000001:2.0:1787764019.262210:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764019.262211:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764019.262212:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc780.
00000100:00000001:2.0:1787764019.262214:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764019.262215:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.262216:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.262219:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764019.262220:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764019.262222:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764019.262224:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764019.262225:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764019.262227:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc3c0 x1874599832683136/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.262232:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764019.262233:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764019.262235:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764019.262236:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764019.262238:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764019.262240:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832683136:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764019.262242:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764019.262243:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764019.262245:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764019.262246:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764019.262248:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764019.262249:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764019.262252:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764019.262254:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787764019.262256:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787764019.262258:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832683136, portal 10
00000100:00000001:2.0:1787764019.262260:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764019.262261:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049846720 : -115096659704896 : ffff9751f87bc3c0)
00000100:00000040:2.0:1787764019.262265:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc3c0 x1874599832683136/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.262269:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764019.262271:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764019.262272:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787764019.262274:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832683136, offset 0
00000400:00000200:2.0:1787764019.262278:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764019.262282:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764019.262286:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764019.262289:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764019.262292:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764019.262294:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764019.262297:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764019.262300:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58572:58572:256:4294967295] 192.168.203.110@tcp LPNI seq info [58572:58572:8:4294967295]
00000400:00000200:2.0:1787764019.262305:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764019.262313:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764019.262318:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764019.262322:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787764019.262326:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764019.262329:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787764019.262333:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764019.262335:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1787764019.262338:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764019.262340:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787764019.262342:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764019.262344:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764019.262346:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787764019.262348:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787764019.262350:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764019.262352:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc3c0 x1874599832683136/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.262358:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764019.262360:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc3c0 x1874599832683136/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764044 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.262364:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764019.262365:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764019.262368:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764019.262370:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764019.262372:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764019.262374:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316406293
00000100:00000001:2.0:1787764019.262375:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316407539
00000100:00000001:2.0:1787764019.262377:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316406293
00000100:00000001:2.0:1787764019.262379:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316407539
00000100:00000001:2.0:1787764019.262380:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764019.262381:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764019.262383:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764019.262385:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764019/real 1787764019]  req@ffff9751f87bc3c0 x1874599832683136/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764019.262391:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764019.262391:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764019.262393:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787764019.262394:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787764019.262395:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787764019.262397:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764019.262399:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc3c0 x1874599832683136/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764019.262404:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc3c0 x1874599832683136/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.262408:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764019.262408:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.262410:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764019.262410:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764019.262413:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc3c0 x1874599832683136/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.262417:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764019.262419:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc3c0 x1874599832683136/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.262423:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764019.262425:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.262426:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764019.262427:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.262428:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764019.262429:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764019.262431:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764019.262433:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764019.262434:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764019.262437:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc3c0 x1874599832683136/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764019.262442:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832683136:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764019.262445:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764019.262447:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bfc00 x1874599832683264/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.262451:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764019.262452:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764019.262453:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764019.262454:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764019.262456:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764019.262457:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832683264:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764019.262459:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764019.262461:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764019.262462:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764019.262463:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764019.262465:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764019.262466:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764019.262469:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764019.262472:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787764019.262475:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787764019.262477:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832683264, portal 4
00000100:00000001:2.0:1787764019.262479:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764019.262480:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049861120 : -115096659690496 : ffff9751f87bfc00)
00000100:00000040:2.0:1787764019.262485:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bfc00 x1874599832683264/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.262489:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764019.262491:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764019.262494:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000200:2.0:1787764019.262495:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832683264, offset 0
00000400:00000200:2.0:1787764019.262499:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764019.262502:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764019.262504:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764019.262506:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764019.262508:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764019.262510:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764019.262511:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764019.262513:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58573:58573:256:4294967295] 192.168.203.110@tcp LPNI seq info [58573:58573:8:4294967295]
00000400:00000200:2.0:1787764019.262517:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764019.262523:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764019.262529:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764019.262532:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000800:00000200:2.0:1787764019.262536:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764019.262539:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787764019.262544:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764019.262547:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1787764019.262554:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764019.262556:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:2.0:1787764019.262560:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764019.262563:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764019.262567:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:2.0:1787764019.262569:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:2.0:1787764019.262571:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764019.262575:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bfc00 x1874599832683264/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764044 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.262584:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764019.262586:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bfc00 x1874599832683264/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764044 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.262591:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764019.262592:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764019.262595:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764019.262597:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764019.262599:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764019.262601:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316406310
00000100:00000001:2.0:1787764019.262603:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316407539
00000100:00000001:2.0:1787764019.262604:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316406310
00000100:00000001:2.0:1787764019.262606:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316407539
00000100:00000001:2.0:1787764019.262607:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764019.262609:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764019.262610:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764019.262612:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764019/real 1787764019]  req@ffff9751f87bfc00 x1874599832683264/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764019.262617:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764019.262618:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764019.262619:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:2.0:1787764019.262621:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787764019.262622:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787764019.262623:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764019.262625:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bfc00 x1874599832683264/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764019.262631:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bfc00 x1874599832683264/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.262634:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764019.262635:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.262636:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764019.262637:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764019.262639:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bfc00 x1874599832683264/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.262643:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764019.262646:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bfc00 x1874599832683264/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.262650:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764019.262652:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.262653:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764019.262653:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.262655:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764019.262656:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764019.262658:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764019.262660:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764019.262661:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764019.262663:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bfc00 x1874599832683264/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764019.262668:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832683264:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764019.262670:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.262672:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764019.262675:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc3c0 x1874599832683136/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.262682:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764019.262683:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764019.262685:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764019.262688:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764019.262689:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764019.262691:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764019.262693:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764019.262694:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612800.
02000000:00000001:2.0:1787764019.262697:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764019.262698:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764019.262700:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc3c0.
00000100:00000001:2.0:1787764019.262702:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764019.262703:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.262705:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764019.262709:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bfc00 x1874599832683264/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764044 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764019.262724:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764019.262725:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764019.262727:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764019.262728:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764019.262729:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764019.262730:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764019.262731:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764019.262733:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613000.
02000000:00000001:2.0:1787764019.262734:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764019.262735:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764019.262736:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bfc00.
00000100:00000001:2.0:1787764019.262738:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764019.262739:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.262740:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764019.262742:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764019.262743:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764019.262744:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764019.262746:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764019.900241:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764019.900243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764019.900249:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764019.900251:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764019.900254:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764019.900255:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764019.900257:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764019.900262:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764019.900262:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764019.900265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764019.900275:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764019.900280:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764019.900284:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:2.0:1787764019.900287:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000100:00000001:1.0:1787764019.900288:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1787764019.900294:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764019.900295:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764019.900301:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764020.284232:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764020.284239:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764020.284241:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764020.284246:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764020.284250:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764020.284253:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764020.284255:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764020.284257:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764020.284259:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764020.540261:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316407539, at 4316407552
00000100:00000010:3.0:1787764020.540291:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb77e0.
00000100:00000040:2.0:1787764020.540318:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316407539, at 4316407552
00000100:00000010:2.0:1787764020.540333:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00010000:00000001:3.0:1787764020.924252:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764020.924271:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764020.924278:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764020.924281:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764020.924285:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764020.924287:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764020.924292:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764020.924299:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764020.924302:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764020.924308:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764020.924314:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764020.924319:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764020.924322:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764020.924325:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764020.924328:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764020.924338:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764020.924343:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764020.924346:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764021.308217:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764021.308226:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764021.308227:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764021.308233:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764021.308237:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764021.308241:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764021.308244:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764021.308246:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764021.308249:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764021.948250:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764021.948259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764021.948262:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764021.948265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764021.948267:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764021.948276:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764021.948277:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764021.948279:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764021.948284:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764021.948285:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764021.948291:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:1.0:1787764021.948292:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764021.948293:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764021.948298:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764021.948299:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764021.948302:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:1.0:1787764021.948304:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764021.948305:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764021.948310:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764021.948313:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764021.948313:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764021.948368:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764021.948372:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764021.948375:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764022.332238:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764022.332248:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764022.332251:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764022.332258:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764022.332263:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764022.332266:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764022.332269:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764022.332271:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764022.332273:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764022.972218:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764022.972228:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764022.972229:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764022.972233:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764022.972235:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764022.972242:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764022.972242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764022.972245:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764022.972250:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764022.972251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764022.972254:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764022.972257:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764022.972262:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764022.972265:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764022.972270:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764022.972301:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764022.972306:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764022.972310:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764023.356266:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764023.356276:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764023.356279:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764023.356284:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764023.356288:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764023.356292:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764023.356295:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764023.356298:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764023.356301:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764023.996223:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764023.996230:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764023.996233:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764023.996238:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764023.996242:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764023.996246:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764023.996249:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764023.996252:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764023.996257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764023.996263:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764023.996264:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764023.996275:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764023.996278:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764023.996299:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764023.996300:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764023.996376:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764023.996380:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764023.996383:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787764024.380222:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764024.380224:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764024.380231:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764024.380233:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764024.380238:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:3.0:1787764024.380241:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787764024.380243:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764024.380245:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787764024.380247:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764024.380251:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:3.0:1787764024.380251:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764024.380256:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764024.380258:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787764024.380260:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764024.380266:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21738
00000400:00000200:3.0:1787764024.380275:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764024.380282:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764024.380285:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764024.380288:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764024.380291:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764024.380292:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764024.380294:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764024.380298:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764024.380300:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787764024.380302:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764024.380305:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787764024.380307:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787764024.380309:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787764024.380311:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764024.380312:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764024.380314:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764024.380315:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764024.380318:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787764024.380320:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764024.380322:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764024.380325:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764024.380327:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764024.380330:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764024.380332:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764024.380334:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764024.380345:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bde00.
00000020:00000040:3.0:1787764024.380348:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787764024.380352:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764024.380353:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764024.380356:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764024.380359:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764024.380363:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764024.380371:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610000.
00000100:00000001:3.0:1787764024.380378:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764024.380387:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bde00 x1874599832683392/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764024.380396:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764024.380401:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764024.380404:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000941163c2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bde00 x1874599832683392/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764024.380423:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764024.380426:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764024.380429:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787764024.380430:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764024.380431:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764024.380434:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764024.380436:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21738
00000400:00000200:3.0:1787764024.380439:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764024.380442:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764024.380443:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787764024.380444:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1787764024.380448:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764024.380449:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764024.380450:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764024.380454:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764024.380455:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764024.380458:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:2.0:1787764024.380462:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bde00 x1874599832683392/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787764024.380462:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764024.380463:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:3.0:1787764024.380467:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764024.380470:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:3.0:1787764024.380475:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787764024.380478:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:3.0:1787764024.380479:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787764024.380481:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764024.380485:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764024.380487:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1787764024.380487:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1787764024.380491:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1787764024.380492:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764024.380497:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764024.380498:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764024.380503:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00100000:2.0:1787764024.380505:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832683392:192.168.203.110@tcp:250:kworker.0
00000100:00080000:3.0:1787764024.380508:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764024.380512:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1787764024.380513:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764024.380515:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764024.380517:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764024.380521:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764024.380522:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764024.380523:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:3.0:1787764024.380526:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764024.380529:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000010:2.0:1787764024.380532:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
00000008:00100000:3.0:1787764024.380533:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1787764024.380535:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1787764024.380538:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764024.380541:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764024.380543:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000010:3.0:1787764024.380544:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bed00.
00000020:00000040:3.0:1787764024.380548:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000010:2.0:1787764024.380550:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
02000000:00000001:3.0:1787764024.380551:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764024.380554:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764024.380556:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764024.380557:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764024.380558:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764024.380560:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
02000000:00000010:3.0:1787764024.380562:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611400.
00000100:00000001:3.0:1787764024.380567:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764024.380569:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832683392, portal 25
00000100:00100000:3.0:1787764024.380572:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bed00 x1874599832683520/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764024.380573:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764024.380576:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049853440 : -115096659698176 : ffff9751f87bde00)
00000100:00000001:3.0:1787764024.380585:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:2.0:1787764024.380586:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bde00 x1874599832683392/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764049 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764024.380589:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764024.380598:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000079751c9a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bed00 x1874599832683520/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764024.380599:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764024.380607:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764024.380612:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecf68.
00000100:00000001:3.0:1787764024.380614:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764024.380615:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832683392, offset 0
00000100:00080000:3.0:1787764024.380619:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787764024.380625:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00080000:3.0:1787764024.380625:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1787764024.380629:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764024.380631:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764024.380633:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764024.380636:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21738
00000400:00000200:2.0:1787764024.380638:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764024.380647:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764024.380652:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764024.380656:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764024.380660:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764024.380663:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764024.380666:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58574:58574:256:4294967295] 192.168.203.110@tcp LPNI seq info [58574:58574:8:4294967295]
00000400:00000200:2.0:1787764024.380670:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764024.380678:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764024.380685:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764024.380685:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764024.380690:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000800:00000010:2.0:1787764024.380692:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fc00.
00000100:00080000:3.0:1787764024.380693:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764024.380697:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764024.380698:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1787764024.380703:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:3.0:1787764024.380703:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000200:2.1:1787764024.380711:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787764024.380712:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764024.380719:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764024.380720:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:3.0:1787764024.380721:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764024.380723:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:3.0:1787764024.380725:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:3.0:1787764024.380727:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:3.0:1787764024.380730:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764024.380731:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764024.380733:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764024.380734:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764024.380735:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764024.380739:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:2.0:1787764024.380741:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:3.0:1787764024.380745:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764024.380747:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316412659
00000100:00000001:3.0:1787764024.380748:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764024.380751:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000010:2.0:1787764024.380754:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
02000000:00000001:3.0:1787764024.380754:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764024.380763:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764024.380765:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764024.380766:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000008:00100000:3.0:1787764024.380768:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787764024.380771:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764024.380773:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764024.380775:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316412659
00000100:00000010:3.0:1787764024.380778:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87be1c0.
00000100:00000010:2.0:1787764024.380781:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb78a0.
00000020:00000040:3.0:1787764024.380782:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000001:2.0:1787764024.380785:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
02000000:00000001:3.0:1787764024.380787:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764024.380790:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764024.380792:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764024.380792:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1787764024.380796:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764024.380796:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764024.380798:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764024.380800:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764024.380804:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
02000000:00000010:3.0:1787764024.380804:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611000.
00000100:00000001:3.0:1787764024.380810:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764024.380814:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00100000:3.0:1787764024.380815:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87be1c0 x1874599832683648/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764024.380817:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1787764024.380822:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764024.380829:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764024.380832:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764024.380832:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764024.380837:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764024.380838:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e2011c6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87be1c0 x1874599832683648/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787764024.380846:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bed00 x1874599832683520/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764024.380848:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764024.380853:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1787764024.380860:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:3.0:1787764024.380866:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764024.380867:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764024.380870:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764024.380872:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
02000000:00000001:2.0:1787764024.380876:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1787764024.380876:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764024.380880:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
02000000:00000001:2.0:1787764024.380881:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764024.380883:0:304484:0:(import.c:485:import_select_connection()) Process entered
02000000:00000001:2.0:1787764024.380885:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764024.380888:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21738
00000100:00100000:2.0:1787764024.380891:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832683520:192.168.203.110@tcp:38:kworker.0
00000400:00000200:3.0:1787764024.380893:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764024.380899:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787764024.380900:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:3.0:1787764024.380904:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
02000000:00000001:2.0:1787764024.380905:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764024.380908:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764024.380911:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:3.0:1787764024.380911:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764024.380916:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764024.380918:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000010:2.0:1787764024.380919:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
00000100:00000001:3.0:1787764024.380920:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:2.0:1787764024.380924:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787764024.380924:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764024.380927:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:3.0:1787764024.380930:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764024.380932:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000400:00000200:2.0:1787764024.380933:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:3.0:1787764024.380935:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:3.0:1787764024.380938:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000010:2.0:1787764024.380939:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000100:00000040:3.0:1787764024.380943:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764024.380946:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1787764024.380947:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000040:3.0:1787764024.380949:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764024.380953:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000200:2.0:1787764024.380955:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832683520, portal 10
00000020:00000040:3.0:1787764024.380955:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:3.0:1787764024.380958:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764024.380964:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764024.380966:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787764024.380970:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:3.0:1787764024.380970:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764024.380973:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049857280 : -115096659694336 : ffff9751f87bed00)
02000000:00000001:3.0:1787764024.380974:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764024.380978:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787764024.380980:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1787764024.380983:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764024.380984:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bed00 x1874599832683520/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764049 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764024.380985:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764024.380991:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc000.
00000020:00000040:3.0:1787764024.380996:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:2.0:1787764024.380997:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:3.0:1787764024.380999:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:2.0:1787764024.381003:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
02000000:00000001:3.0:1787764024.381003:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764024.381007:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:2.0:1787764024.381008:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49eca18.
02000000:00000001:3.0:1787764024.381009:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764024.381013:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764024.381014:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832683520, offset 0
02000000:00000010:3.0:1787764024.381017:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613400.
00000400:00000200:2.0:1787764024.381022:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787764024.381024:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764024.381028:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00100000:3.0:1787764024.381030:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc000 x1874599832683776/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787764024.381035:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764024.381042:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1787764024.381044:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1787764024.381047:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:3.0:1787764024.381048:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764024.381052:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000040:3.0:1787764024.381055:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3fc1c6c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc000 x1874599832683776/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764024.381057:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:3.0:1787764024.381073:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764024.381079:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58575:58575:255:4294967295] 192.168.203.110@tcp LPNI seq info [58575:58575:7:4294967295]
00000100:00000040:3.0:1787764024.381080:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21749)
00000400:00000200:2.0:1787764024.381097:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764024.381103:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764024.381107:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764024.381111:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646ff00.
00000800:00000200:2.0:1787764024.381114:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764024.381118:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787764024.381120:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764024.381123:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764024.381126:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764024.381127:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316411423
00000100:00000001:2.0:1787764024.381129:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316412659
00000100:00000001:2.0:1787764024.381131:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316411423
00000100:00000001:2.0:1787764024.381133:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316412659
00000100:00000001:2.0:1787764024.381151:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764024.381152:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764024.381154:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764024.381155:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764024.381158:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764024.381159:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1787764024.381161:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764024.381162:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764024.381163:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764024.381167:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87be1c0 x1874599832683648/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764024.381176:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764024.381177:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764024.381179:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764024.381181:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764024.381183:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764024.381187:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832683648:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764024.381190:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764024.381192:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764024.381194:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764024.381196:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764024.381200:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764024.381202:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764024.381206:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764024.381209:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da04d0.
00000400:00000010:2.0:1787764024.381213:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec5d8.
00000100:00000200:2.0:1787764024.381222:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832683648, portal 10
00000100:00000001:2.0:1787764024.381225:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764024.381226:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049854400 : -115096659697216 : ffff9751f87be1c0)
00000100:00000040:2.0:1787764024.381233:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87be1c0 x1874599832683648/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764049 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764024.381240:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764024.381243:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764024.381246:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec3b8.
00000100:00000200:2.0:1787764024.381248:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832683648, offset 0
00000400:00000200:2.0:1787764024.381253:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764024.381258:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764024.381262:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764024.381265:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764024.381268:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764024.381271:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764024.381273:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764024.381276:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58576:58576:254:4294967295] 192.168.203.110@tcp LPNI seq info [58576:58576:6:4294967295]
00000400:00000200:2.0:1787764024.381281:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764024.381286:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764024.381291:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764024.381294:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f400.
00000800:00000200:2.0:1787764024.381298:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764024.381302:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787764024.381305:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764024.381308:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764024.381311:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764024.381313:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316411412
00000100:00000001:2.0:1787764024.381316:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316412659
00000100:00000001:2.0:1787764024.381318:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316411412
00000100:00000001:2.0:1787764024.381321:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316412659
00000100:00000001:2.0:1787764024.381323:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764024.381325:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764024.381328:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764024.381331:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc000 x1874599832683776/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764024.381340:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764024.381341:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764024.381344:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764024.381345:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764024.381347:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764024.381350:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832683776:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764024.381354:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764024.381356:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764024.381357:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764024.381359:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764024.381364:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764024.381366:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764024.381370:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764024.381373:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000400:00000010:2.0:1787764024.381377:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764024.381380:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832683776, portal 4
00000100:00000001:2.0:1787764024.381382:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764024.381383:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049845760 : -115096659705856 : ffff9751f87bc000)
00000100:00000040:2.0:1787764024.381389:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc000 x1874599832683776/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764049 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764024.381395:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764024.381397:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764024.381400:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec000.
00000100:00000200:2.0:1787764024.381402:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832683776, offset 0
00000400:00000200:2.0:1787764024.381407:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764024.381411:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764024.381414:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764024.381418:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764024.381420:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764024.381423:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764024.381425:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764024.381427:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58577:58577:253:4294967295] 192.168.203.110@tcp LPNI seq info [58577:58577:5:4294967295]
00000400:00000200:2.0:1787764024.381431:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764024.381437:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764024.381441:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764024.381445:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f200.
00000800:00000200:2.0:1787764024.381448:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764024.381452:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787764024.381454:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764024.381457:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764024.381459:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764024.381461:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316411429
00000100:00000001:2.0:1787764024.381464:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316412659
00000100:00000001:2.0:1787764024.381465:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316411429
00000100:00000001:2.0:1787764024.381468:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316412659
00000100:00000001:2.0:1787764024.381470:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764024.381472:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764024.381474:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764024.381476:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764024.381480:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764024.381481:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1787764024.381483:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764024.381484:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764024.381486:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764024.381487:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764025.020219:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764025.020231:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764025.020234:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764025.020238:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764025.020238:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764025.020242:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764025.020246:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764025.020247:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764025.020251:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:1.0:1787764025.020253:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764025.020254:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764025.020257:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764025.020258:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764025.020260:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764025.020262:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764025.020265:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764025.020266:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764025.020268:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764025.020270:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764025.020272:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764025.020274:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764025.020353:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764025.020358:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764025.020361:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764025.404264:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764025.404274:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764025.404277:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764025.404283:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764025.404289:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764025.404294:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764025.404297:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764025.404301:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764025.404305:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764025.660275:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316412659, at 4316412672
00000100:00000010:3.0:1787764025.660293:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb78a0.
00000100:00000040:2.0:1787764025.660334:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316412659, at 4316412672
00000100:00000010:2.0:1787764025.660343:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00000100:00000001:1.0:1787764026.044247:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764026.044251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764026.044256:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764026.044257:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764026.044260:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764026.044264:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764026.044265:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764026.044267:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764026.044270:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764026.044273:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764026.044275:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764026.044276:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764026.044280:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764026.044284:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764026.044286:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764026.044316:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764026.044321:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764026.044324:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764026.428255:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764026.428267:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764026.428271:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764026.428280:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764026.428285:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764026.428290:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764026.428294:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764026.428298:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764026.428302:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764027.068215:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764027.068215:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764027.068222:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764027.068224:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764027.068229:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764027.068231:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764027.068234:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764027.068236:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:1.0:1787764027.068239:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764027.068240:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:1.0:1787764027.068245:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764027.068246:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764027.068251:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764027.068252:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:1.0:1787764027.068254:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764027.068256:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764027.068259:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764027.068261:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764027.068266:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764027.068271:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764027.068273:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764027.068316:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764027.068320:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764027.068323:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764027.452245:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764027.452252:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764027.452254:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764027.452258:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764027.452261:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764027.452264:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764027.452266:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764027.452268:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764027.452271:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:0.0:1787764027.452404:0:257566:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.110:988
00000400:00000100:0.0:1787764027.452440:0:257566:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.110@tcp at host 192.168.203.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:0.0:1787764027.452449:0:257566:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764027.452452:0:257566:0:(router.c:1791:lnet_notify()) 192.168.203.10@tcp notifying 192.168.203.110@tcp: down
00000400:00000200:0.0:1787764027.452455:0:257566:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764027.452458:0:257566:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:0.0:1787764027.452462:0:257566:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:0.0:1787764027.452465:0:257566:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fc00.
00000400:00000200:0.0:1787764027.452469:0:257566:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1787764027.452472:0:257566:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1787764027.452479:0:257566:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecf68
00000400:00000010:0.0:1787764027.452481:0:257566:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecf68.
00000100:00000001:0.0:1787764027.452487:0:257566:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787764027.452496:0:257566:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f87bde00 x1874599832683392/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764049 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764027.452520:0:257566:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787764027.452525:0:257566:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bde00 x1874599832683392/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764049 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764027.452533:0:257566:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764027.452535:0:257566:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:0.0:1787764027.452547:0:257566:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:0.0:1787764027.452550:0:257566:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646ff00.
00000100:00000001:2.0:1787764027.452553:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:0.0:1787764027.452554:0:257566:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00000001:2.0:1787764027.452558:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000400:00000200:0.0:1787764027.452562:0:257566:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000100:00000001:2.0:1787764027.452563:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000400:00000200:0.0:1787764027.452569:0:257566:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49eca18
00000100:00100000:2.0:1787764027.452574:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764024/real 1787764027]  req@ffff9751f87bde00 x1874599832683392/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1787764027.452575:0:257566:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49eca18.
00000100:00000001:0.0:1787764027.452580:0:257566:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787764027.452588:0:257566:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f87bed00 x1874599832683520/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764049 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764027.452595:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:0.0:1787764027.452599:0:257566:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000400:00000001:2.0:1787764027.452600:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764027.452606:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000100:00000040:0.0:1787764027.452607:0:257566:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bed00 x1874599832683520/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764049 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764027.452613:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787764027.452618:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:0.0:1787764027.452621:0:257566:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764027.452624:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:0.0:1787764027.452625:0:257566:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000200:2.0:1787764027.452633:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bde00 x1874599832683392/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:0.0:1787764027.452636:0:257566:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:0.0:1787764027.452643:0:257566:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f400.
00000400:00000200:0.0:1787764027.452648:0:257566:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00000200:2.0:1787764027.452657:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bde00 x1874599832683392/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787764027.452658:0:257566:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1787764027.452665:0:257566:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec3b8
00000400:00000010:0.0:1787764027.452669:0:257566:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec3b8.
00000100:00000001:2.0:1787764027.452674:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787764027.452675:0:257566:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:2.0:1787764027.452678:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000200:0.0:1787764027.452684:0:257566:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f87be1c0 x1874599832683648/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764049 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764027.452685:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764027.452689:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764027.452698:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bde00 x1874599832683392/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764027.452702:0:257566:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787764027.452709:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787764027.452715:0:257566:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87be1c0 x1874599832683648/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764049 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787764027.452719:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bde00 x1874599832683392/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764027.452738:0:257566:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764027.452741:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787764027.452743:0:257566:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764027.452748:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764027.452753:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000800:00000100:0.0:1787764027.452754:0:257566:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000100:00000001:2.0:1787764027.452756:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000800:00000010:0.0:1787764027.452760:0:257566:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f200.
00000100:00000001:2.0:1787764027.452762:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764027.452767:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000400:00000200:0.0:1787764027.452769:0:257566:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00080000:2.0:1787764027.452771:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000400:00000200:0.0:1787764027.452777:0:257566:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000100:00080000:2.0:1787764027.452781:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000400:00000200:0.0:1787764027.452782:0:257566:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec000
00000100:00000001:2.0:1787764027.452786:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000400:00000010:0.0:1787764027.452787:0:257566:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec000.
00000100:00000001:0.0:1787764027.452794:0:257566:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:2.0:1787764027.452797:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bde00 x1874599832683392/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787764027.452802:0:257566:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f87bc000 x1874599832683776/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764049 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764027.452824:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832683392:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787764027.452826:0:257566:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787764027.452833:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764027.452834:0:257566:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc000 x1874599832683776/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764049 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764027.452839:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1787764027.452848:0:257566:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764027.452848:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bde00 x1874599832683392/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764027.452854:0:257566:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764027.452862:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764027.452866:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764027.452870:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764027.452876:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764027.452879:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764027.452881:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764027.452884:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764027.452887:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610000.
02000000:00000001:2.0:1787764027.452890:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764027.452893:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764027.452895:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bde00.
00000100:00000001:2.0:1787764027.452899:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764027.452901:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764027.452904:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764027.452921:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764027.452923:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764027.452924:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764027.452930:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764024/real 1787764027]  req@ffff9751f87be1c0 x1874599832683648/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764027.452940:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764027.452942:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764027.452945:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da04d0.
00000400:00000200:2.0:1787764027.452948:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec5d8
00000400:00000010:2.0:1787764027.452949:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec5d8.
00000100:00000001:2.0:1787764027.452952:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764027.452956:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87be1c0 x1874599832683648/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764027.452966:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87be1c0 x1874599832683648/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764027.452973:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764027.452974:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764027.452977:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764027.452978:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764027.452982:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87be1c0 x1874599832683648/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764027.452988:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764027.452992:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87be1c0 x1874599832683648/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764027.452998:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764027.453001:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764027.453003:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764027.453004:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764027.453005:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764027.453007:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764027.453009:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764027.453013:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764027.453015:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764027.453019:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87be1c0 x1874599832683648/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764027.453026:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832683648:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764027.453030:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764027.453033:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764024/real 1787764027]  req@ffff9751f87bc000 x1874599832683776/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764027.453056:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764027.453057:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764027.453059:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:2.0:1787764027.453062:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764027.453064:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764027.453066:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764027.453071:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc000 x1874599832683776/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764027.453078:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc000 x1874599832683776/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764027.453085:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764027.453086:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764027.453088:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764027.453089:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764027.453093:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc000 x1874599832683776/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764027.453099:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764027.453104:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc000 x1874599832683776/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764027.453111:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764027.453113:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764027.453115:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764027.453116:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764027.453118:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764027.453120:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764027.453123:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764027.453127:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764027.453129:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764027.453155:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc000 x1874599832683776/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764027.453166:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832683776:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764027.453172:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764027.453178:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764024/real 1787764027]  req@ffff9751f87bed00 x1874599832683520/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764027.453193:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764027.453194:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764027.453196:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764027.453199:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764027.453201:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764027.453203:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764027.453206:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bed00 x1874599832683520/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764027.453215:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bed00 x1874599832683520/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764027.453221:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764027.453223:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764027.453224:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764027.453226:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764027.453229:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bed00 x1874599832683520/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764027.453235:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764027.453239:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bed00 x1874599832683520/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764027.453246:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764027.453249:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764027.453251:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764027.453252:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764027.453254:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764027.453269:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764027.453272:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764027.453276:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764027.453279:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764027.453284:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bed00 x1874599832683520/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764027.453292:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832683520:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764027.453297:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764027.453299:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764027.453302:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87be1c0 x1874599832683648/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764027.453309:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764027.453311:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764027.453314:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764027.453317:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764027.453319:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764027.453321:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764027.453323:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764027.453325:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611000.
02000000:00000001:2.0:1787764027.453328:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764027.453329:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764027.453331:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87be1c0.
00000100:00000001:2.0:1787764027.453334:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764027.453335:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764027.453337:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764027.453346:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc000 x1874599832683776/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764027.453353:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764027.453354:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764027.453356:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764027.453359:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764027.453360:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764027.453366:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764027.453368:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764027.453370:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613400.
02000000:00000001:2.0:1787764027.453373:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764027.453375:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764027.453376:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc000.
00000100:00000001:2.0:1787764027.453379:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764027.453380:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764027.453381:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764027.453385:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bed00 x1874599832683520/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764049 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764027.453391:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764027.453392:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764027.453393:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764027.453396:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764027.453397:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764027.453398:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764027.453400:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764027.453402:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611400.
02000000:00000001:2.0:1787764027.453404:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764027.453405:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764027.453406:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bed00.
00000100:00000001:2.0:1787764027.453408:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764027.453409:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764027.453411:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764027.453419:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764027.453421:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764027.453423:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764027.453424:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764028.092217:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764028.092223:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764028.092228:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764028.092229:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764028.092230:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764028.092233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764028.092236:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764028.092237:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764028.092242:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764028.092246:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764028.092251:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764028.092255:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764028.092259:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764028.092261:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764028.092264:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764028.092291:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764028.092295:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764028.092298:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764028.476221:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764028.476229:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764028.476232:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764028.476238:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764028.476242:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764028.476246:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764028.476249:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764028.476251:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764028.476255:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764029.116203:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764029.116210:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764029.116212:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764029.116214:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764029.116218:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764029.116218:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764029.116223:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764029.116227:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764029.116227:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764029.116232:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764029.116232:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764029.116236:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764029.116239:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764029.116240:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764029.116244:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764029.116300:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764029.116304:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764029.116307:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764029.500204:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764029.500211:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764029.500213:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764029.500217:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764029.500220:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764029.500223:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.500225:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:3.0:1787764029.500227:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764029.500228:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.500232:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764029.500238:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764029.500242:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764029.500246:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764029.500251:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764029.500260:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21744
00000400:00000200:3.0:1787764029.500269:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764029.500276:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764029.500279:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764029.500281:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764029.500284:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764029.500286:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764029.500287:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764029.500290:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764029.500292:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787764029.500295:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764029.500298:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787764029.500300:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787764029.500302:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787764029.500304:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764029.500305:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764029.500306:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764029.500307:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764029.500309:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787764029.500312:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764029.500314:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764029.500316:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764029.500319:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764029.500321:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764029.500322:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764029.500324:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764029.500329:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bf480.
00000020:00000040:3.0:1787764029.500332:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787764029.500335:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764029.500336:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764029.500338:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764029.500341:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764029.500344:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764029.500350:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613000.
00000100:00000001:3.0:1787764029.500355:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764029.500362:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bf480 x1874599832683904/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764029.500369:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764029.500373:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764029.500378:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c2965388] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bf480 x1874599832683904/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764029.500401:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764029.500405:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764029.500408:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787764029.500409:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764029.500411:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764029.500414:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764029.500417:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21744
00000400:00000200:3.0:1787764029.500421:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764029.500425:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764029.500428:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787764029.500430:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1787764029.500431:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764029.500435:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764029.500436:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764029.500439:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764029.500442:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764029.500447:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:2.0:1787764029.500450:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bf480 x1874599832683904/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787764029.500451:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764029.500454:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:3.0:1787764029.500458:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764029.500461:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000001:2.0:1787764029.500471:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:3.0:1787764029.500472:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787764029.500474:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787764029.500476:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:2.0:1787764029.500480:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:3.0:1787764029.500482:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764029.500485:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764029.500485:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764029.500490:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764029.500491:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764029.500493:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764029.500497:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00100000:2.0:1787764029.500501:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832683904:192.168.203.110@tcp:250:kworker.0
00000100:00080000:3.0:1787764029.500501:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764029.500508:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764029.500511:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764029.500511:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764029.500517:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764029.500520:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764029.500521:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764029.500527:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:3.0:1787764029.500527:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764029.500532:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000010:2.0:1787764029.500535:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
00000008:00100000:3.0:1787764029.500537:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1787764029.500539:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1787764029.500545:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764029.500550:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764029.500550:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764029.500557:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000100:00000010:3.0:1787764029.500558:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bed00.
00000020:00000040:3.0:1787764029.500563:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787764029.500566:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:2.0:1787764029.500567:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
02000000:00000001:3.0:1787764029.500569:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764029.500573:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764029.500575:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:2.0:1787764029.500576:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832683904, portal 25
02000000:00000001:3.0:1787764029.500578:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764029.500581:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764029.500585:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049859200 : -115096659692416 : ffff9751f87bf480)
02000000:00000010:3.0:1787764029.500586:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612800.
00000100:00000001:3.0:1787764029.500594:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764029.500596:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bf480 x1874599832683904/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764054 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764029.500604:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bed00 x1874599832684032/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764029.500622:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764029.500629:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787764029.500634:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:2.0:1787764029.500635:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000001:3.0:1787764029.500639:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764029.500641:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832683904, offset 0
00000100:00000040:3.0:1787764029.500648:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000079751c9a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bed00 x1874599832684032/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764029.500654:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787764029.500664:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764029.500666:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:3.0:1787764029.500669:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764029.500675:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:2.0:1787764029.500676:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:3.0:1787764029.500678:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764029.500682:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1787764029.500686:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1787764029.500688:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:2.0:1787764029.500692:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00080000:3.0:1787764029.500696:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21744
00000400:00000200:2.0:1787764029.500701:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764029.500705:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764029.500709:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58578:58578:256:4294967295] 192.168.203.110@tcp LPNI seq info [58578:58578:8:4294967295]
00000400:00000200:2.0:1787764029.500715:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764029.500725:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000400:00000200:3.0:1787764029.500734:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:2.0:1787764029.500735:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764029.500744:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000800:00000010:2.0:1787764029.500745:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000100:00080000:3.0:1787764029.500749:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000200:2.0:1787764029.500758:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:3.0:1787764029.500758:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764029.500765:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1787764029.500766:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000100:00000040:3.0:1787764029.500769:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764029.500775:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:2.1:1787764029.500778:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:3.0:1787764029.500780:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764029.500783:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000800:00000200:2.1:1787764029.500785:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000020:00000001:3.0:1787764029.500787:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764029.500792:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000100:2.0:1787764029.500797:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000020:00000001:3.0:1787764029.500798:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000010:2.0:1787764029.500802:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000020:00000001:3.0:1787764029.500803:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:2.0:1787764029.500811:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1787764029.500811:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764029.500818:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764029.500824:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764029.500825:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1787764029.500830:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787764029.500834:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000020:00000040:3.0:1787764029.500835:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000010:2.0:1787764029.500840:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00080000:3.0:1787764029.500842:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764029.500846:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:3.0:1787764029.500848:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764029.500852:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:2.0:1787764029.500855:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bf480 x1874599832683904/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764054 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787764029.500857:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764029.500869:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:3.0:1787764029.500869:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764029.500874:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:2.0:1787764029.500875:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bf480 x1874599832683904/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764054 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:3.0:1787764029.500877:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787764029.500883:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764029.500888:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764029.500896:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc000.
00000020:00000040:3.0:1787764029.500902:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000001:2.0:1787764029.500903:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764029.500908:0:257575:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:3.0:1787764029.500908:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764029.500912:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764029.500914:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1787764029.500916:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764029.500917:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764029.500922:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764029.500926:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000010:3.0:1787764029.500930:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612c00.
00000100:00000001:2.0:1787764029.500931:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316417779
00000100:00000001:3.0:1787764029.500937:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764029.500940:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00100000:3.0:1787764029.500946:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc000 x1874599832684160/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764029.500950:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787764029.500959:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316417779
00000100:00000010:2.0:1787764029.500964:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7840.
00000100:00000001:2.0:1787764029.500968:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787764029.500972:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764029.500973:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764029.500976:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764029.500978:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764029.500979:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764029.500988:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764029/real 1787764029]  req@ffff9751f87bf480 x1874599832683904/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787764029.500989:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3fc1c6c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc000 x1874599832684160/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764029.501004:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764029.501009:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:3.0:1787764029.501009:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764029.501014:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1787764029.501019:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000010:2.0:1787764029.501020:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000100:00080000:3.0:1787764029.501035:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787764029.501037:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764029.501042:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00080000:3.0:1787764029.501043:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787764029.501047:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:3.0:1787764029.501047:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764029.501052:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:2.0:1787764029.501055:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bf480 x1874599832683904/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764029.501057:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764029.501063:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21744
00000100:00000200:2.0:1787764029.501075:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bf480 x1874599832683904/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764029.501078:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764029.501086:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787764029.501090:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00080000:3.0:1787764029.501091:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787764029.501094:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764029.501098:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764029.501099:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764029.501103:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764029.501104:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764029.501109:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00100000:2.0:1787764029.501113:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bf480 x1874599832683904/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764029.501114:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764029.501119:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764029.501126:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000100:00000001:2.0:1787764029.501149:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764029.501155:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bf480 x1874599832683904/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.501160:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764029.501162:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.501163:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764029.501164:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.501166:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764029.501168:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764029.501169:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764029.501172:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764029.501174:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764029.501177:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bf480 x1874599832683904/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764029.501182:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832683904:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764029.501185:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.501187:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764029.501189:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bf480 x1874599832683904/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.501194:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764029.501195:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764029.501197:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764029.501200:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764029.501202:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764029.501204:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764029.501206:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764029.501208:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613000.
02000000:00000001:2.0:1787764029.501211:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764029.501213:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764029.501215:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bf480.
00000100:00000001:2.0:1787764029.501219:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764029.501220:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.501222:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.501230:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764029.501233:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787764029.501235:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1787764029.501237:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:3.0:1787764029.501240:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000001:2.0:1787764029.501241:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764029.501244:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1787764029.501246:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:2.0:1787764029.501251:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bed00 x1874599832684032/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787764029.501252:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:3.0:1787764029.501262:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764029.501268:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764029.501269:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764029.501272:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764029.501274:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764029.501277:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1787764029.501279:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:2.0:1787764029.501281:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764029.501286:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1787764029.501286:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00100000:2.0:1787764029.501296:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832684032:192.168.203.110@tcp:38:kworker.0
00000100:00080000:3.0:1787764029.501299:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764029.501300:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764029.501304:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1787764029.501306:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764029.501307:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764029.501311:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764029.501312:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:3.0:1787764029.501317:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000010:2.0:1787764029.501318:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764029.501323:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764029.501325:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764029.501330:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:2.0:1787764029.501331:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000008:00100000:3.0:1787764029.501335:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000400:00000010:2.0:1787764029.501336:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000008:00000001:3.0:1787764029.501342:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764029.501343:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000001:3.0:1787764029.501345:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764029.501351:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87be1c0.
00000100:00000200:2.0:1787764029.501354:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832684032, portal 10
00000020:00000040:3.0:1787764029.501356:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:2.0:1787764029.501359:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:3.0:1787764029.501362:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1787764029.501363:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049857280 : -115096659694336 : ffff9751f87bed00)
02000000:00000001:3.0:1787764029.501366:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764029.501370:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:2.0:1787764029.501374:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bed00 x1874599832684032/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764054 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787764029.501374:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764029.501377:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764029.501385:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612400.
00000100:00000001:2.0:1787764029.501387:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764029.501392:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787764029.501392:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764029.501399:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764029.501403:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832684032, offset 0
00000100:00100000:3.0:1787764029.501404:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87be1c0 x1874599832684288/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787764029.501412:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764029.501421:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764029.501425:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1787764029.501428:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:3.0:1787764029.501431:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764029.501434:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764029.501440:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000040:3.0:1787764029.501440:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e2011c6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87be1c0 x1874599832684288/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764029.501445:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764029.501450:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764029.501458:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58579:58579:256:4294967295] 192.168.203.110@tcp LPNI seq info [58579:58579:8:4294967295]
00000100:00000001:3.0:1787764029.501463:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764029.501465:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:3.0:1787764029.501471:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21754)
00000400:00000200:2.0:1787764029.501474:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764029.501480:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764029.501484:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764029.501489:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764029.501492:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787764029.501496:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764029.501499:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787764029.501503:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764029.501505:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764029.501508:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764029.501512:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764029.501515:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764029.501517:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764029.501520:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764029.501523:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bed00 x1874599832684032/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764054 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.501530:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764029.501533:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bed00 x1874599832684032/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764054 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.501540:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764029.501541:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764029.501546:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764029.501550:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764029.501553:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764029.501555:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316416544
00000100:00000001:2.0:1787764029.501557:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316417779
00000100:00000001:2.0:1787764029.501559:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316416544
00000100:00000001:2.0:1787764029.501561:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316417779
00000100:00000001:2.0:1787764029.501563:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764029.501565:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764029.501567:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764029.501570:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764029/real 1787764029]  req@ffff9751f87bed00 x1874599832684032/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764029.501586:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764029.501588:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764029.501590:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764029.501592:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764029.501594:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764029.501596:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764029.501599:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bed00 x1874599832684032/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764029.501608:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bed00 x1874599832684032/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.501615:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764029.501617:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.501619:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764029.501620:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764029.501624:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bed00 x1874599832684032/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.501632:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764029.501636:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bed00 x1874599832684032/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.501645:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764029.501648:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.501650:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764029.501651:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.501653:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764029.501667:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764029.501670:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764029.501674:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764029.501677:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764029.501681:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bed00 x1874599832684032/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764029.501689:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832684032:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764029.501694:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764029.501698:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc000 x1874599832684160/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.501704:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764029.501706:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764029.501708:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764029.501710:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764029.501712:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764029.501715:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832684160:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764029.501719:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764029.501721:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764029.501723:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764029.501725:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764029.501729:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764029.501731:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764029.501735:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764029.501738:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764029.501741:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764029.501744:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832684160, portal 10
00000100:00000001:2.0:1787764029.501745:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764029.501746:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049845760 : -115096659705856 : ffff9751f87bc000)
00000100:00000040:2.0:1787764029.501752:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc000 x1874599832684160/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764054 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.501759:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764029.501761:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764029.501764:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764029.501766:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832684160, offset 0
00000400:00000200:2.0:1787764029.501771:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764029.501775:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764029.501779:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764029.501782:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764029.501785:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764029.501788:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764029.501790:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764029.501793:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58580:58580:256:4294967295] 192.168.203.110@tcp LPNI seq info [58580:58580:8:4294967295]
00000400:00000200:2.0:1787764029.501797:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764029.501804:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764029.501809:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764029.501813:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764029.501816:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764029.501819:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787764029.501824:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764029.501826:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787764029.501831:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764029.501833:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764029.501836:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764029.501839:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764029.501843:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764029.501846:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764029.501848:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764029.501852:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc000 x1874599832684160/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764054 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.501858:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764029.501860:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc000 x1874599832684160/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764054 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.501864:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764029.501865:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764029.501868:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764029.501870:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764029.501873:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764029.501874:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316416532
00000100:00000001:2.0:1787764029.501876:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316417779
00000100:00000001:2.0:1787764029.501878:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316416532
00000100:00000001:2.0:1787764029.501879:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316417779
00000100:00000001:2.0:1787764029.501881:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764029.501882:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764029.501883:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764029.501886:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764029/real 1787764029]  req@ffff9751f87bc000 x1874599832684160/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764029.501891:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764029.501892:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764029.501893:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764029.501894:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764029.501895:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764029.501897:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764029.501899:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc000 x1874599832684160/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764029.501904:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc000 x1874599832684160/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.501909:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764029.501910:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.501911:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764029.501912:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764029.501914:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc000 x1874599832684160/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.501918:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764029.501920:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc000 x1874599832684160/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.501924:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764029.501926:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.501927:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764029.501927:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.501929:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764029.501930:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764029.501931:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764029.501933:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764029.501934:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764029.501937:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc000 x1874599832684160/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764029.501941:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832684160:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764029.501944:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.501945:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764029.501947:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bed00 x1874599832684032/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.501952:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764029.501953:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764029.501954:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764029.501956:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764029.501958:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764029.501959:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764029.501961:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764029.501963:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612800.
02000000:00000001:2.0:1787764029.501966:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764029.501967:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764029.501968:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bed00.
00000100:00000001:2.0:1787764029.501971:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764029.501972:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.501974:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764029.501977:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc000 x1874599832684160/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.501984:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764029.501985:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764029.501987:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764029.501989:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764029.501990:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764029.501991:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764029.501993:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764029.501995:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612c00.
02000000:00000001:2.0:1787764029.501998:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764029.501999:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764029.502000:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc000.
00000100:00000001:2.0:1787764029.502003:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764029.502004:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.502006:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.502010:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764029.502012:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764029.502014:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764029.502016:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764029.502017:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764029.502020:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87be1c0 x1874599832684288/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.502040:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764029.502043:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764029.502044:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764029.502046:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764029.502048:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764029.502051:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832684288:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764029.502054:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764029.502056:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764029.502057:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764029.502059:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764029.502062:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764029.502064:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764029.502067:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764029.502069:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764029.502073:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764029.502075:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832684288, portal 4
00000100:00000001:2.0:1787764029.502077:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764029.502078:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049854400 : -115096659697216 : ffff9751f87be1c0)
00000100:00000040:2.0:1787764029.502084:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87be1c0 x1874599832684288/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764054 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.502090:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764029.502092:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764029.502094:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764029.502096:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832684288, offset 0
00000400:00000200:2.0:1787764029.502099:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764029.502103:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764029.502106:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764029.502109:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764029.502116:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764029.502118:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764029.502120:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764029.502123:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58581:58581:256:4294967295] 192.168.203.110@tcp LPNI seq info [58581:58581:8:4294967295]
00000400:00000200:2.0:1787764029.502127:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764029.502154:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764029.502161:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764029.502164:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764029.502168:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764029.502171:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:2.1:1787764029.502175:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764029.502178:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:2.0:1787764029.502182:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764029.502184:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764029.502187:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764029.502191:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764029.502194:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764029.502196:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764029.502199:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764029.502202:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87be1c0 x1874599832684288/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764054 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.502211:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764029.502214:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87be1c0 x1874599832684288/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764054 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.502220:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764029.502222:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764029.502226:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764029.502229:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764029.502233:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764029.502235:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316416550
00000100:00000001:2.0:1787764029.502238:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316417779
00000100:00000001:2.0:1787764029.502241:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316416550
00000100:00000001:2.0:1787764029.502243:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316417779
00000100:00000001:2.0:1787764029.502245:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764029.502248:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764029.502250:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764029.502254:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764029/real 1787764029]  req@ffff9751f87be1c0 x1874599832684288/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764029.502265:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764029.502266:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764029.502268:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764029.502270:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764029.502272:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764029.502274:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764029.502278:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87be1c0 x1874599832684288/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764029.502287:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87be1c0 x1874599832684288/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.502294:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764029.502296:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.502298:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764029.502300:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764029.502303:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87be1c0 x1874599832684288/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.502308:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764029.502310:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87be1c0 x1874599832684288/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.502315:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764029.502316:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.502317:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764029.502318:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.502319:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764029.502320:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764029.502322:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764029.502324:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764029.502325:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764029.502328:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87be1c0 x1874599832684288/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764029.502333:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832684288:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764029.502336:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.502337:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764029.502339:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87be1c0 x1874599832684288/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764054 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764029.502343:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764029.502344:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764029.502345:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764029.502347:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764029.502347:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764029.502348:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764029.502349:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764029.502350:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612400.
02000000:00000001:2.0:1787764029.502352:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764029.502353:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764029.502354:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87be1c0.
00000100:00000001:2.0:1787764029.502355:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764029.502356:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.502357:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764029.502360:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764029.502361:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764029.502362:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764029.502363:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764030.140242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764030.140248:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764030.140250:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764030.140254:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764030.140256:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764030.140257:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764030.140258:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764030.140262:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764030.140264:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764030.140265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764030.140270:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764030.140273:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764030.140276:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764030.140278:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764030.140280:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764030.140316:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764030.140321:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764030.140325:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764030.524222:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764030.524230:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764030.524232:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764030.524237:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764030.524241:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764030.524256:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764030.524262:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764030.524265:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764030.524269:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764030.780252:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316417779, at 4316417792
00000100:00000010:3.0:1787764030.780267:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7840.
00000100:00000040:2.0:1787764030.780324:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316417779, at 4316417792
00000100:00000010:2.0:1787764030.780337:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00000100:00000001:1.0:1787764031.164212:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764031.164222:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764031.164225:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764031.164230:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764031.164233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764031.164235:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764031.164237:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764031.164239:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764031.164243:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764031.164243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764031.164246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764031.164247:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764031.164251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764031.164252:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764031.164253:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764031.164309:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764031.164314:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764031.164318:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764031.548231:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764031.548254:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764031.548256:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764031.548261:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764031.548265:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764031.548269:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764031.548271:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764031.548273:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764031.548276:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764032.188230:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764032.188232:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764032.188237:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764032.188238:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764032.188241:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764032.188242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764032.188247:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764032.188251:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764032.188254:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764032.188254:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764032.188260:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764032.188262:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764032.188263:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764032.188264:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764032.188267:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764032.188270:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764032.188273:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764032.188275:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764032.188277:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764032.188278:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764032.188279:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764032.188313:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764032.188316:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764032.188319:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764032.572246:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764032.572254:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764032.572257:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764032.572263:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764032.572268:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764032.572287:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764032.572290:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764032.572292:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764032.572295:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764033.212280:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764033.212300:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764033.212308:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764033.212310:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764033.212315:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764033.212318:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764033.212322:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764033.212323:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764033.212333:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764033.212334:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764033.212340:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764033.212340:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764033.212349:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764033.212354:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764033.212358:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764033.212415:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764033.212419:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764033.212422:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764033.596266:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764033.596279:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764033.596283:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764033.596293:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764033.596302:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764033.596309:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764033.596315:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764033.596320:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764033.596329:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764034.236229:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764034.236231:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764034.236236:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764034.236237:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764034.236241:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764034.236243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764034.236245:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764034.236246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764034.236251:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764034.236253:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764034.236255:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764034.236256:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764034.236261:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764034.236264:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764034.236266:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764034.236321:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764034.236326:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764034.236330:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764034.620209:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764034.620218:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764034.620222:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764034.620228:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764034.620234:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764034.620238:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.620240:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764034.620242:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.620245:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764034.620252:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764034.620262:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787764034.620264:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764034.620267:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764034.620271:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764034.620277:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21749
00000400:00000200:1.0:1787764034.620285:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764034.620291:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764034.620294:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764034.620297:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764034.620299:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764034.620301:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764034.620303:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764034.620306:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764034.620308:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787764034.620310:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764034.620313:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787764034.620314:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787764034.620316:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787764034.620318:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764034.620320:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764034.620321:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764034.620322:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764034.620325:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787764034.620327:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764034.620329:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764034.620331:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764034.620334:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764034.620336:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764034.620338:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764034.620340:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764034.620348:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933840.
00000020:00000040:1.0:1787764034.620351:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787764034.620353:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764034.620354:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764034.620357:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764034.620359:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764034.620362:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764034.620370:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb400.
00000100:00000001:1.0:1787764034.620375:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764034.620382:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933840 x1874599832684416/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764034.620390:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764034.620393:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764034.620396:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000083dd6d32] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933840 x1874599832684416/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764034.620410:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764034.620413:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764034.620415:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787764034.620416:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764034.620417:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764034.620419:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764034.620422:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21749
00000400:00000200:1.0:1787764034.620425:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764034.620428:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764034.620429:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764034.620431:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764034.620433:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764034.620433:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:1.0:1787764034.620438:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764034.620438:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764034.620440:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764034.620442:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1787764034.620443:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764034.620446:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787764034.620447:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764034.620451:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000040:2.0:1787764034.620451:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933840 x1874599832684416/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1787764034.620455:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787764034.620459:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787764034.620466:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764034.620469:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764034.620470:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764034.620473:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764034.620474:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764034.620477:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:1.0:1787764034.620478:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:2.0:1787764034.620482:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:1.0:1787764034.620483:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:2.0:1787764034.620488:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764034.620489:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764034.620493:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764034.620495:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832684416:192.168.203.110@tcp:250:kworker.0
02000000:00000001:1.0:1787764034.620496:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787764034.620501:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764034.620503:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764034.620504:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764034.620506:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764034.620508:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764034.620509:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1787764034.620512:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:1.0:1787764034.620514:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787764034.620518:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764034.620518:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
00000100:00000001:1.0:1787764034.620524:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764034.620524:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764034.620532:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932580.
00000400:00000200:2.0:1787764034.620533:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:1.0:1787764034.620535:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000010:2.0:1787764034.620538:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
02000000:00000001:1.0:1787764034.620539:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764034.620541:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764034.620544:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764034.620546:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764034.620547:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764034.620547:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
02000000:00000010:1.0:1787764034.620551:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9000.
00000100:00000200:2.0:1787764034.620555:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832684416, portal 25
00000100:00000001:1.0:1787764034.620556:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764034.620560:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764034.620563:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497735744 : -115097211815872 : ffff9751d7933840)
00000100:00100000:1.0:1787764034.620565:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932580 x1874599832684544/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764034.620573:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:2.0:1787764034.620574:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933840 x1874599832684416/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764059 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764034.620575:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764034.620590:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d026924] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932580 x1874599832684544/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.620599:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:1.0:1787764034.620601:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764034.620606:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1787764034.620607:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00080000:1.0:1787764034.620612:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787764034.620613:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:2.0:1787764034.620614:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00080000:1.0:1787764034.620615:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764034.620620:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:2.0:1787764034.620621:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832684416, offset 0
00000100:00080000:1.0:1787764034.620623:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21749
00000400:00000200:1.0:1787764034.620628:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764034.620632:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1787764034.620632:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00080000:1.0:1787764034.620638:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764034.620640:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764034.620642:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1787764034.620645:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:1.0:1787764034.620646:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764034.620650:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764034.620652:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764034.620653:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787764034.620655:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:2.0:1787764034.620656:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000020:00000040:1.0:1787764034.620657:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787764034.620662:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:2.0:1787764034.620663:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:1.0:1787764034.620665:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:2.0:1787764034.620670:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000040:1.0:1787764034.620671:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764034.620673:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1787764034.620676:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000040:1.0:1787764034.620677:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764034.620680:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787764034.620681:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000020:00000040:1.0:1787764034.620685:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:2.0:1787764034.620688:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58582:58582:256:4294967295] 192.168.203.110@tcp LPNI seq info [58582:58582:8:4294967295]
00000100:00080000:1.0:1787764034.620692:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764034.620697:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764034.620697:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:1.0:1787764034.620701:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764034.620704:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764034.620709:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764034.620713:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:2.0:1787764034.620714:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000008:00100000:1.0:1787764034.620717:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787764034.620720:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764034.620723:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787764034.620723:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000010:1.0:1787764034.620729:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930780.
00000020:00000040:1.0:1787764034.620733:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000800:00000010:2.0:1787764034.620733:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
02000000:00000001:1.0:1787764034.620738:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764034.620740:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764034.620741:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764034.620742:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764034.620744:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787764034.620747:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000010:1.0:1787764034.620749:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8000.
00000100:00000001:1.0:1787764034.620753:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787764034.620754:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000100:00100000:1.0:1787764034.620760:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930780 x1874599832684672/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.1:1787764034.620767:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764034.620775:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:1.0:1787764034.620778:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000100:2.0:1787764034.620782:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787764034.620785:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1787764034.620786:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764034.620793:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:1.0:1787764034.620795:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3977950] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930780 x1874599832684672/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764034.620800:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764034.620807:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764034.620810:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:1.0:1787764034.620814:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764034.620815:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:1.0:1787764034.620819:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:2.0:1787764034.620822:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933840 x1874599832684416/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764059 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787764034.620824:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787764034.620830:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764034.620836:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787764034.620837:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787764034.620838:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764034.620842:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:2.0:1787764034.620844:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933840 x1874599832684416/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764059 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764034.620845:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764034.620851:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21749
00000400:00000200:1.0:1787764034.620860:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764034.620866:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787764034.620869:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764034.620870:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787764034.620873:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:1.0:1787764034.620876:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764034.620879:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764034.620881:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764034.620882:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764034.620887:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787764034.620891:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:1.0:1787764034.620892:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764034.620895:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000100:00000001:2.0:1787764034.620895:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316422899
00000020:00000001:1.0:1787764034.620897:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764034.620901:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787764034.620902:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000010:2.0:1787764034.620903:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000020:00000001:1.0:1787764034.620904:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:1.0:1787764034.620909:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764034.620910:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764034.620911:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000040:1.0:1787764034.620913:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764034.620916:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764034.620918:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:2.0:1787764034.620919:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316422899
00000100:00080000:1.0:1787764034.620921:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000010:2.0:1787764034.620924:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb76c0.
00000100:00000001:1.0:1787764034.620925:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764034.620929:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787764034.620929:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
02000000:00000001:1.0:1787764034.620935:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764034.620935:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764034.620939:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764034.620940:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764034.620944:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:1.0:1787764034.620945:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764034.620949:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1787764034.620953:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764034.620953:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764034/real 1787764034]  req@ffff9751d7933840 x1874599832684416/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764034.620958:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764034.620965:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932940.
00000400:00000001:2.0:1787764034.620966:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000040:1.0:1787764034.620968:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000400:00000001:2.0:1787764034.620969:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:1.0:1787764034.620971:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:2.0:1787764034.620972:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
02000000:00000001:1.0:1787764034.620974:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764034.620976:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000100:00000001:1.0:1787764034.620977:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:2.0:1787764034.620979:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
02000000:00000001:1.0:1787764034.620980:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787764034.620982:0:257575:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:1.0:1787764034.620984:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764034.621000:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933840 x1874599832684416/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1787764034.621005:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca800.
00000100:00000001:1.0:1787764034.621010:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764034.621014:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933840 x1874599832684416/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764034.621019:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932940 x1874599832684800/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764034.621024:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764034.621027:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.621031:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764034.621032:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764034.621034:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764034.621036:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764034.621041:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933840 x1874599832684416/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787764034.621043:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000af90eaa] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932940 x1874599832684800/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621062:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764034.621066:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764034.621071:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933840 x1874599832684416/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787764034.621072:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21759)
00000100:00000001:2.0:1787764034.621083:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764034.621087:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.621089:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764034.621091:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.621093:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764034.621095:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764034.621098:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764034.621103:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764034.621106:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764034.621111:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933840 x1874599832684416/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764034.621120:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832684416:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764034.621124:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.621125:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764034.621128:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933840 x1874599832684416/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621155:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764034.621158:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764034.621160:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764034.621164:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764034.621166:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764034.621168:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764034.621171:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764034.621173:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb400.
02000000:00000001:2.0:1787764034.621177:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764034.621179:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764034.621180:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933840.
00000100:00000001:2.0:1787764034.621184:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764034.621185:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.621188:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.621195:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764034.621197:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764034.621199:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764034.621201:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764034.621202:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764034.621207:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932580 x1874599832684544/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621215:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764034.621217:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764034.621219:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764034.621221:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764034.621223:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764034.621227:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832684544:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764034.621231:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764034.621232:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764034.621234:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764034.621235:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764034.621237:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764034.621238:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764034.621242:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764034.621244:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764034.621246:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764034.621249:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832684544, portal 10
00000100:00000001:2.0:1787764034.621250:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764034.621251:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497730944 : -115097211820672 : ffff9751d7932580)
00000100:00000040:2.0:1787764034.621256:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932580 x1874599832684544/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764059 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621260:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764034.621262:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764034.621264:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764034.621265:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832684544, offset 0
00000400:00000200:2.0:1787764034.621268:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764034.621271:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764034.621274:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764034.621276:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764034.621278:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764034.621280:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764034.621282:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764034.621284:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58583:58583:256:4294967295] 192.168.203.110@tcp LPNI seq info [58583:58583:8:4294967295]
00000400:00000200:2.0:1787764034.621286:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764034.621291:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764034.621294:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764034.621296:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764034.621299:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764034.621301:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787764034.621304:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764034.621307:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1787764034.621309:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764034.621311:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764034.621313:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764034.621315:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764034.621317:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764034.621318:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764034.621320:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764034.621322:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932580 x1874599832684544/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764059 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621328:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764034.621330:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932580 x1874599832684544/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764059 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621334:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764034.621335:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764034.621338:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764034.621340:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764034.621343:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764034.621344:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316421664
00000100:00000001:2.0:1787764034.621346:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316422899
00000100:00000001:2.0:1787764034.621348:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316421664
00000100:00000001:2.0:1787764034.621349:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316422899
00000100:00000001:2.0:1787764034.621350:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764034.621351:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764034.621353:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764034.621355:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764034/real 1787764034]  req@ffff9751d7932580 x1874599832684544/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764034.621360:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764034.621361:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764034.621362:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764034.621363:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764034.621364:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764034.621366:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764034.621368:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932580 x1874599832684544/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764034.621373:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932580 x1874599832684544/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621377:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764034.621378:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.621379:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764034.621380:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764034.621382:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932580 x1874599832684544/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621386:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764034.621388:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932580 x1874599832684544/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621394:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764034.621395:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.621396:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764034.621397:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.621398:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764034.621407:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764034.621409:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764034.621411:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764034.621412:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764034.621415:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932580 x1874599832684544/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764034.621420:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832684544:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764034.621423:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764034.621425:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930780 x1874599832684672/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621428:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764034.621429:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764034.621431:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764034.621432:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764034.621433:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764034.621435:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832684672:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764034.621436:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764034.621438:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764034.621439:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764034.621440:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764034.621442:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764034.621443:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764034.621445:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764034.621446:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764034.621448:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764034.621450:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832684672, portal 10
00000100:00000001:2.0:1787764034.621452:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764034.621453:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497723264 : -115097211828352 : ffff9751d7930780)
00000100:00000040:2.0:1787764034.621456:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930780 x1874599832684672/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764059 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621460:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764034.621462:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764034.621463:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764034.621465:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832684672, offset 0
00000400:00000200:2.0:1787764034.621467:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764034.621470:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764034.621472:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764034.621473:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764034.621475:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764034.621476:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764034.621478:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764034.621480:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58584:58584:256:4294967295] 192.168.203.110@tcp LPNI seq info [58584:58584:8:4294967295]
00000400:00000200:2.0:1787764034.621483:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764034.621487:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764034.621490:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764034.621491:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764034.621493:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764034.621495:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787764034.621498:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764034.621499:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1787764034.621502:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764034.621503:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764034.621505:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764034.621507:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764034.621509:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764034.621510:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764034.621512:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764034.621514:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930780 x1874599832684672/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764059 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621519:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764034.621521:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930780 x1874599832684672/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764059 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621525:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764034.621526:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764034.621528:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764034.621530:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764034.621532:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764034.621533:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316421652
00000100:00000001:2.0:1787764034.621535:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316422899
00000100:00000001:2.0:1787764034.621536:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316421652
00000100:00000001:2.0:1787764034.621537:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316422899
00000100:00000001:2.0:1787764034.621539:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764034.621540:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764034.621541:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764034.621543:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764034/real 1787764034]  req@ffff9751d7930780 x1874599832684672/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764034.621547:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764034.621548:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764034.621549:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764034.621550:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764034.621552:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764034.621553:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764034.621555:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930780 x1874599832684672/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764034.621560:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930780 x1874599832684672/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621565:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764034.621566:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.621568:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764034.621569:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764034.621573:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930780 x1874599832684672/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621580:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764034.621584:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930780 x1874599832684672/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621591:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764034.621594:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.621595:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764034.621597:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.621599:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764034.621600:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764034.621603:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764034.621606:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764034.621609:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764034.621613:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930780 x1874599832684672/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764034.621620:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832684672:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764034.621623:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764034.621625:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932940 x1874599832684800/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621629:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764034.621630:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764034.621632:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764034.621633:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764034.621635:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764034.621637:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832684800:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764034.621639:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764034.621640:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764034.621641:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764034.621642:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764034.621645:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764034.621646:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764034.621649:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764034.621651:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764034.621653:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764034.621655:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832684800, portal 4
00000100:00000001:2.0:1787764034.621656:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764034.621657:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497731904 : -115097211819712 : ffff9751d7932940)
00000100:00000040:2.0:1787764034.621662:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932940 x1874599832684800/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764059 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621666:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764034.621667:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764034.621668:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764034.621670:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832684800, offset 0
00000400:00000200:2.0:1787764034.621672:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764034.621675:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764034.621677:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764034.621679:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764034.621681:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764034.621682:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764034.621684:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764034.621686:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58585:58585:256:4294967295] 192.168.203.110@tcp LPNI seq info [58585:58585:8:4294967295]
00000400:00000200:2.0:1787764034.621689:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764034.621693:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764034.621696:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764034.621698:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764034.621701:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764034.621703:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787764034.621705:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764034.621707:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1787764034.621710:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764034.621711:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764034.621713:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764034.621715:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764034.621717:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764034.621718:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764034.621720:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764034.621722:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932940 x1874599832684800/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764059 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621727:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764034.621730:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932940 x1874599832684800/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764059 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621736:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764034.621738:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764034.621741:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764034.621744:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764034.621747:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764034.621750:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316421669
00000100:00000001:2.0:1787764034.621752:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316422899
00000100:00000001:2.0:1787764034.621754:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316421669
00000100:00000001:2.0:1787764034.621757:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316422899
00000100:00000001:2.0:1787764034.621759:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764034.621761:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764034.621763:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764034.621768:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764034/real 1787764034]  req@ffff9751d7932940 x1874599832684800/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764034.621777:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764034.621778:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764034.621780:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764034.621782:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764034.621784:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764034.621785:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764034.621787:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932940 x1874599832684800/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764034.621793:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932940 x1874599832684800/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621797:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764034.621798:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.621799:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764034.621800:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764034.621802:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932940 x1874599832684800/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621807:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764034.621809:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932940 x1874599832684800/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621814:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764034.621815:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.621816:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764034.621817:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.621818:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764034.621819:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764034.621821:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764034.621823:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764034.621824:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764034.621827:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932940 x1874599832684800/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764034.621832:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832684800:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764034.621835:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.621836:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764034.621838:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932580 x1874599832684544/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621842:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764034.621843:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764034.621844:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764034.621846:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764034.621847:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764034.621848:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764034.621849:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764034.621851:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9000.
02000000:00000001:2.0:1787764034.621852:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764034.621854:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764034.621855:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932580.
00000100:00000001:2.0:1787764034.621857:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764034.621858:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.621859:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764034.621861:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930780 x1874599832684672/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621865:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764034.621866:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764034.621867:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764034.621868:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764034.621869:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764034.621869:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764034.621870:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764034.621871:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8000.
02000000:00000001:2.0:1787764034.621872:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764034.621874:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764034.621875:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930780.
00000100:00000001:2.0:1787764034.621876:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764034.621877:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.621878:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764034.621880:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932940 x1874599832684800/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764059 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764034.621884:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764034.621885:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764034.621885:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764034.621887:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764034.621887:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764034.621888:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764034.621889:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764034.621890:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca800.
02000000:00000001:2.0:1787764034.621891:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764034.621892:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764034.621893:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932940.
00000100:00000001:2.0:1787764034.621894:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764034.621894:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.621895:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764034.621898:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764034.621899:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764034.621900:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764034.621901:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764035.260240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764035.260242:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764035.260248:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764035.260249:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:1.0:1787764035.260252:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764035.260253:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764035.260257:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764035.260258:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764035.260262:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:1.0:1787764035.260263:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764035.260265:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764035.260267:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764035.260269:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764035.260272:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764035.260273:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764035.260274:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764035.260278:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764035.260279:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764035.260281:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764035.260286:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764035.260286:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764035.260324:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764035.260328:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764035.260330:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764035.644264:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764035.644274:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764035.644277:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764035.644285:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764035.644304:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764035.644308:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764035.644311:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764035.644313:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764035.644316:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764035.900252:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316422899, at 4316422912
00000100:00000010:3.0:1787764035.900267:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb76c0.
00000100:00000040:2.0:1787764035.900316:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316422899, at 4316422912
00000100:00000010:2.0:1787764035.900327:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00000100:00000001:1.0:1787764036.284228:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764036.284228:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764036.284233:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764036.284235:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764036.284238:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764036.284240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764036.284242:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764036.284243:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764036.284249:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764036.284250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764036.284252:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764036.284253:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764036.284257:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764036.284258:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764036.284261:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764036.284290:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764036.284293:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764036.284297:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764036.668273:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764036.668284:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764036.668288:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764036.668296:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764036.668301:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764036.668306:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764036.668311:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764036.668315:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764036.668320:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764037.308201:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764037.308203:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764037.308208:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764037.308208:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764037.308212:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764037.308214:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764037.308216:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764037.308219:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:1.0:1787764037.308221:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764037.308222:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:1.0:1787764037.308225:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764037.308227:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764037.308229:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764037.308231:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:1.0:1787764037.308232:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764037.308234:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764037.308237:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764037.308238:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764037.308244:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764037.308248:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764037.308250:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764037.308318:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764037.308322:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764037.308324:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764037.692252:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764037.692258:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764037.692260:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764037.692265:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764037.692269:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764037.692271:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764037.692273:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764037.692276:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764037.692278:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764038.332291:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764038.332302:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764038.332305:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764038.332310:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764038.332313:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764038.332321:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764038.332324:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764038.332333:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764038.332338:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764038.332341:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764038.332345:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764038.332350:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764038.332357:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764038.332366:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764038.332369:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764038.332460:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764038.332468:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764038.332475:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764038.716198:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764038.716205:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764038.716207:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764038.716212:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764038.716215:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764038.716219:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764038.716221:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764038.716223:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764038.716225:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764039.356269:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764039.356271:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764039.356280:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764039.356281:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764039.356286:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764039.356290:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764039.356295:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764039.356295:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764039.356301:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764039.356304:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764039.356305:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764039.356313:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764039.356320:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764039.356324:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764039.356328:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764039.356361:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764039.356367:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764039.356371:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764039.740234:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:1.0:1787764039.740239:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764039.740241:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764039.740245:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764039.740251:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764039.740256:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:1.0:1787764039.740261:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787764039.740261:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764039.740267:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764039.740269:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764039.740273:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787764039.740273:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764039.740282:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764039.740289:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764039.740299:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21754
00000400:00000200:1.0:1787764039.740306:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764039.740312:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764039.740315:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764039.740317:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764039.740320:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764039.740322:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764039.740324:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764039.740327:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764039.740329:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787764039.740331:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764039.740335:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787764039.740337:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787764039.740339:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787764039.740341:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764039.740343:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764039.740345:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764039.740346:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764039.740348:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787764039.740351:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764039.740353:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764039.740356:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764039.740358:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764039.740360:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764039.740362:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764039.740364:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764039.740373:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933480.
00000020:00000040:1.0:1787764039.740376:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787764039.740379:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764039.740381:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764039.740384:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764039.740387:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764039.740392:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764039.740402:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb800.
00000100:00000001:1.0:1787764039.740410:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764039.740422:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933480 x1874599832684928/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764039.740436:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764039.740441:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764039.740447:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3c2c33c] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933480 x1874599832684928/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764039.740466:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764039.740469:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764039.740472:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787764039.740473:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764039.740475:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764039.740477:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764039.740480:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21754
00000400:00000200:1.0:1787764039.740485:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764039.740488:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787764039.740490:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:1.0:1787764039.740491:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787764039.740496:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1787764039.740497:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764039.740501:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1787764039.740502:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764039.740507:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764039.740509:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764039.740513:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000040:2.0:1787764039.740513:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933480 x1874599832684928/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787764039.740516:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787764039.740520:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764039.740523:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787764039.740527:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787764039.740533:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787764039.740535:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:1.0:1787764039.740539:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764039.740539:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764039.740544:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1787764039.740545:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:1.0:1787764039.740546:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764039.740550:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:2.0:1787764039.740550:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:1.0:1787764039.740558:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:2.0:1787764039.740558:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764039.740566:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00100000:2.0:1787764039.740569:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832684928:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787764039.740571:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764039.740577:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787764039.740578:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764039.740583:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764039.740585:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764039.740586:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764039.740594:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764039.740594:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1787764039.740600:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000010:2.0:1787764039.740602:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
00000008:00100000:1.0:1787764039.740606:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1787764039.740606:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:1.0:1787764039.740615:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764039.740615:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764039.740622:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764039.740622:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000100:00000010:1.0:1787764039.740633:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930000.
00000400:00000010:2.0:1787764039.740634:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000020:00000040:1.0:1787764039.740642:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000100:00000200:2.0:1787764039.740645:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832684928, portal 25
00000100:00000001:2.0:1787764039.740650:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:1.0:1787764039.740651:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1787764039.740652:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497734784 : -115097211816832 : ffff9751d7933480)
02000000:00000001:1.0:1787764039.740654:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764039.740658:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764039.740662:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:2.0:1787764039.740662:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933480 x1874599832684928/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764064 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787764039.740666:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764039.740677:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8800.
00000100:00000001:2.0:1787764039.740685:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:1.0:1787764039.740686:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764039.740691:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00100000:1.0:1787764039.740700:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930000 x1874599832685056/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:2.0:1787764039.740707:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764039.740713:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832684928, offset 0
00000100:00000001:1.0:1787764039.740721:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764039.740727:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764039.740727:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000040:1.0:1787764039.740736:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6d90274] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930000 x1874599832685056/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764039.740742:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764039.740752:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764039.740753:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00080000:1.0:1787764039.740759:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787764039.740762:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:1.0:1787764039.740783:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787764039.740786:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:2.0:1787764039.740788:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00080000:1.0:1787764039.740790:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1787764039.740795:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1787764039.740797:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:2.0:1787764039.740801:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00080000:1.0:1787764039.740805:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21754
00000400:00000200:2.0:1787764039.740812:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58586:58586:256:4294967295] 192.168.203.110@tcp LPNI seq info [58586:58586:8:4294967295]
00000400:00000200:1.0:1787764039.740816:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764039.740823:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764039.740825:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764039.740829:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764039.740835:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764039.740842:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1787764039.740843:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:1.0:1787764039.740852:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000200:2.0:1787764039.740856:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787764039.740861:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000010:2.0:1787764039.740865:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000020:00000001:1.0:1787764039.740867:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764039.740873:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787764039.740875:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764039.740879:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000200:2.0:1787764039.740882:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:1.0:1787764039.740885:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:2.0:1787764039.740888:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000020:00000001:1.0:1787764039.740893:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:2.1:1787764039.740897:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764039.740902:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000100:00000040:1.0:1787764039.740905:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000100:2.0:1787764039.740912:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787764039.740913:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000010:2.0:1787764039.740918:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000100:00000040:1.0:1787764039.740920:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764039.740924:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787764039.740927:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:1.0:1787764039.740928:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787764039.740936:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:2.0:1787764039.740941:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:1.0:1787764039.740943:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764039.740947:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1787764039.740949:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764039.740966:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
02000000:00000001:1.0:1787764039.740968:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764039.740971:0:257575:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:1.0:1787764039.740974:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764039.740979:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933480 x1874599832684928/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764064 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764039.740980:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764039.740985:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787764039.741009:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764039.741017:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764039.741026:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000010:1.0:1787764039.741027:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930b40.
00000020:00000040:1.0:1787764039.741036:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000040:2.0:1787764039.741038:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933480 x1874599832684928/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764064 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787764039.741044:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764039.741049:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764039.741055:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764039.741060:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764039.741064:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764039.741076:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8c00.
00000100:00000001:2.0:1787764039.741079:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764039.741084:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764039.741085:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764039.741098:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764039.741100:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930b40 x1874599832685184/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787764039.741125:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764039.741131:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316428019
00000100:00000001:1.0:1787764039.741133:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764039.741138:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764039.741140:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00000040:1.0:1787764039.741142:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022d7f91a] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930b40 x1874599832685184/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.741150:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787764039.741152:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764039.741156:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764039.741159:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316428019
00000100:00000040:1.0:1787764039.741160:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000010:2.0:1787764039.741185:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb72a0.
00000100:00000001:2.0:1787764039.741189:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787764039.741193:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:1.0:1787764039.741194:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764039.741198:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764039.741200:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787764039.741203:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787764039.741203:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00080000:1.0:1787764039.741207:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764039.741211:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00100000:2.0:1787764039.741212:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764039/real 1787764039]  req@ffff9751d7933480 x1874599832684928/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787764039.741217:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21754
00000400:00000200:1.0:1787764039.741228:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000001:2.0:1787764039.741234:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000200:1.0:1787764039.741236:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000001:2.0:1787764039.741237:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764039.741241:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000100:00080000:1.0:1787764039.741244:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1787764039.741245:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764039.741251:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000040:1.0:1787764039.741254:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764039.741256:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:1.0:1787764039.741263:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:2.0:1787764039.741263:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933480 x1874599832684928/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787764039.741271:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764039.741275:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764039.741282:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764039.741284:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787764039.741290:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000200:2.0:1787764039.741295:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933480 x1874599832684928/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787764039.741297:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787764039.741302:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787764039.741307:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:2.0:1787764039.741315:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000040:1.0:1787764039.741317:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764039.741319:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764039.741320:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764039.741323:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:1.0:1787764039.741325:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764039.741327:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764039.741329:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764039.741334:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00100000:2.0:1787764039.741335:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933480 x1874599832684928/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787764039.741340:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764039.741346:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764039.741352:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787764039.741355:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:1.0:1787764039.741357:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764039.741363:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764039.741364:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933480 x1874599832684928/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764039.741367:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764039.741372:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:2.0:1787764039.741377:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000008:00000001:1.0:1787764039.741378:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764039.741385:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764039.741385:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764039.741389:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000010:1.0:1787764039.741391:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7931680.
00000100:00000001:2.0:1787764039.741391:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764039.741393:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764039.741397:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000040:1.0:1787764039.741399:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00080000:2.0:1787764039.741401:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000001:1.0:1787764039.741408:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00080000:2.0:1787764039.741409:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
02000000:00000001:1.0:1787764039.741414:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764039.741415:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:1.0:1787764039.741421:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:2.0:1787764039.741423:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933480 x1874599832684928/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787764039.741427:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764039.741432:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764039.741438:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832684928:192.168.203.110@tcp:250:kworker.0
02000000:00000010:1.0:1787764039.741442:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca400.
00000100:00000001:2.0:1787764039.741443:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764039.741448:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787764039.741451:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764039.741453:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933480 x1874599832684928/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764039.741461:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7931680 x1874599832685312/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764039.741463:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764039.741467:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764039.741472:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
00000100:00000001:1.0:1787764039.741474:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:2.0:1787764039.741477:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000001:1.0:1787764039.741478:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764039.741482:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000100:00000040:1.0:1787764039.741485:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d4255d6] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7931680 x1874599832685312/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787764039.741486:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764039.741492:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764039.741496:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb800.
00000100:00000001:1.0:1787764039.741501:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764039.741503:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764039.741506:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000040:1.0:1787764039.741508:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21764)
00000100:00000010:2.0:1787764039.741509:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933480.
00000100:00000001:2.0:1787764039.741516:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764039.741518:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764039.741519:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764039.741526:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764039.741528:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764039.741530:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764039.741532:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764039.741533:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764039.741537:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930000 x1874599832685056/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.741543:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764039.741544:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764039.741546:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764039.741548:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764039.741549:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764039.741552:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832685056:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764039.741554:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764039.741556:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764039.741557:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764039.741558:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764039.741561:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764039.741562:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764039.741565:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764039.741567:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764039.741570:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764039.741572:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832685056, portal 10
00000100:00000001:2.0:1787764039.741574:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764039.741575:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497721344 : -115097211830272 : ffff9751d7930000)
00000100:00000040:2.0:1787764039.741580:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930000 x1874599832685056/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764064 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.741586:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764039.741588:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764039.741590:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764039.741591:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832685056, offset 0
00000400:00000200:2.0:1787764039.741594:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764039.741598:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764039.741601:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764039.741604:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764039.741606:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764039.741608:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764039.741610:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764039.741612:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58587:58587:256:4294967295] 192.168.203.110@tcp LPNI seq info [58587:58587:8:4294967295]
00000400:00000200:2.0:1787764039.741615:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764039.741621:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764039.741624:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764039.741627:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764039.741630:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764039.741632:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787764039.741636:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764039.741638:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1787764039.741641:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764039.741642:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764039.741645:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764039.741648:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764039.741650:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764039.741651:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764039.741653:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764039.741656:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930000 x1874599832685056/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764064 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.741662:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764039.741664:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930000 x1874599832685056/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764064 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.741670:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764039.741671:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764039.741674:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764039.741677:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764039.741679:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764039.741681:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316426784
00000100:00000001:2.0:1787764039.741683:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316428019
00000100:00000001:2.0:1787764039.741685:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316426784
00000100:00000001:2.0:1787764039.741687:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316428019
00000100:00000001:2.0:1787764039.741689:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764039.741691:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764039.741692:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764039.741695:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764039/real 1787764039]  req@ffff9751d7930000 x1874599832685056/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764039.741700:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764039.741701:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764039.741702:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764039.741704:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764039.741705:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764039.741706:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764039.741708:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930000 x1874599832685056/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764039.741714:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930000 x1874599832685056/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.741719:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764039.741721:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764039.741722:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764039.741723:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764039.741725:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930000 x1874599832685056/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.741730:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764039.741733:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930000 x1874599832685056/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.741739:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764039.741741:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764039.741742:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764039.741743:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764039.741744:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764039.741772:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764039.741774:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764039.741777:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764039.741779:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764039.741782:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930000 x1874599832685056/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764039.741789:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832685056:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764039.741792:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764039.741795:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930b40 x1874599832685184/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.741800:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764039.741801:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764039.741803:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764039.741804:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764039.741806:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764039.741808:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832685184:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764039.741810:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764039.741812:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764039.741813:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764039.741815:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764039.741817:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764039.741819:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764039.741822:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764039.741824:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764039.741826:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764039.741829:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832685184, portal 10
00000100:00000001:2.0:1787764039.741831:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764039.741832:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497724224 : -115097211827392 : ffff9751d7930b40)
00000100:00000040:2.0:1787764039.741837:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930b40 x1874599832685184/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764064 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.741843:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764039.741845:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764039.741847:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764039.741848:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832685184, offset 0
00000400:00000200:2.0:1787764039.741852:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764039.741854:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764039.741857:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764039.741859:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764039.741862:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764039.741864:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764039.741866:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764039.741868:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58588:58588:256:4294967295] 192.168.203.110@tcp LPNI seq info [58588:58588:8:4294967295]
00000400:00000200:2.0:1787764039.741872:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764039.741877:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764039.741881:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764039.741883:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764039.741886:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764039.741888:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787764039.741891:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764039.741893:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1787764039.741897:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764039.741899:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764039.741901:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764039.741904:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764039.741906:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764039.741908:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764039.741910:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764039.741913:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930b40 x1874599832685184/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764064 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.741919:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764039.741922:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930b40 x1874599832685184/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764064 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.741927:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764039.741929:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764039.741931:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764039.741933:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764039.741936:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764039.741937:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316426772
00000100:00000001:2.0:1787764039.741940:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316428019
00000100:00000001:2.0:1787764039.741942:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316426772
00000100:00000001:2.0:1787764039.741943:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316428019
00000100:00000001:2.0:1787764039.741945:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764039.741946:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764039.741948:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764039.741951:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764039/real 1787764039]  req@ffff9751d7930b40 x1874599832685184/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764039.741972:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764039.741974:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764039.741975:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764039.741977:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764039.741978:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764039.741980:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764039.741983:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930b40 x1874599832685184/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764039.741990:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930b40 x1874599832685184/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.742013:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764039.742015:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764039.742016:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764039.742018:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764039.742022:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930b40 x1874599832685184/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.742028:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764039.742031:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930b40 x1874599832685184/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.742037:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764039.742039:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764039.742041:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764039.742042:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764039.742044:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764039.742045:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764039.742048:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764039.742051:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764039.742053:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764039.742057:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930b40 x1874599832685184/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764039.742063:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832685184:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764039.742067:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764039.742070:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7931680 x1874599832685312/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.742076:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764039.742077:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764039.742079:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764039.742081:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764039.742083:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764039.742086:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832685312:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764039.742089:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764039.742091:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764039.742092:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764039.742094:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764039.742098:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764039.742100:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764039.742121:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764039.742123:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764039.742126:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764039.742128:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832685312, portal 4
00000100:00000001:2.0:1787764039.742130:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764039.742131:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497727104 : -115097211824512 : ffff9751d7931680)
00000100:00000040:2.0:1787764039.742135:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7931680 x1874599832685312/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764064 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.742140:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764039.742142:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764039.742143:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764039.742145:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832685312, offset 0
00000400:00000200:2.0:1787764039.742148:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764039.742151:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764039.742153:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764039.742155:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764039.742157:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764039.742159:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764039.742161:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764039.742163:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58589:58589:256:4294967295] 192.168.203.110@tcp LPNI seq info [58589:58589:8:4294967295]
00000400:00000200:2.0:1787764039.742183:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764039.742188:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764039.742192:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764039.742195:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764039.742198:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764039.742200:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787764039.742203:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764039.742206:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1787764039.742209:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764039.742210:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764039.742213:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764039.742215:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764039.742217:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764039.742219:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764039.742221:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764039.742224:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7931680 x1874599832685312/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764064 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.742230:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764039.742232:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7931680 x1874599832685312/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764064 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.742237:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764039.742238:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764039.742240:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764039.742243:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764039.742245:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764039.742247:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316426790
00000100:00000001:2.0:1787764039.742249:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316428019
00000100:00000001:2.0:1787764039.742251:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316426790
00000100:00000001:2.0:1787764039.742252:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316428019
00000100:00000001:2.0:1787764039.742254:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764039.742255:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764039.742257:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764039.742259:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764039/real 1787764039]  req@ffff9751d7931680 x1874599832685312/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764039.742266:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764039.742267:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764039.742268:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764039.742270:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764039.742271:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764039.742272:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764039.742274:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7931680 x1874599832685312/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764039.742281:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7931680 x1874599832685312/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.742286:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764039.742287:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764039.742288:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764039.742289:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764039.742292:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7931680 x1874599832685312/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.742297:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764039.742299:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7931680 x1874599832685312/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.742304:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764039.742306:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764039.742307:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764039.742308:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764039.742309:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764039.742310:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764039.742312:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764039.742315:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764039.742316:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764039.742319:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7931680 x1874599832685312/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764039.742326:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832685312:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764039.742328:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764039.742330:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764039.742332:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930000 x1874599832685056/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.742336:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764039.742338:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764039.742339:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764039.742341:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764039.742342:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764039.742343:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764039.742344:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764039.742346:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8800.
02000000:00000001:2.0:1787764039.742347:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764039.742348:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764039.742349:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930000.
00000100:00000001:2.0:1787764039.742351:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764039.742352:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764039.742354:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764039.742356:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930b40 x1874599832685184/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.742361:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764039.742362:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764039.742363:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764039.742364:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764039.742365:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764039.742366:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764039.742367:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764039.742368:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8c00.
02000000:00000001:2.0:1787764039.742370:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764039.742371:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764039.742372:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930b40.
00000100:00000001:2.0:1787764039.742374:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764039.742375:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764039.742376:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764039.742378:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7931680 x1874599832685312/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764064 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764039.742383:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764039.742384:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764039.742385:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764039.742386:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764039.742387:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764039.742388:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764039.742389:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764039.742390:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca400.
02000000:00000001:2.0:1787764039.742392:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764039.742392:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764039.742393:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7931680.
00000100:00000001:2.0:1787764039.742395:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764039.742396:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764039.742397:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764039.742400:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764039.742401:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764039.742402:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764039.742403:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764040.380217:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764040.380221:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764040.380224:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764040.380226:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764040.380228:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764040.380232:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764040.380233:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764040.380236:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764040.380237:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764040.380242:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764040.380243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764040.380245:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764040.380246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764040.380249:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764040.380252:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764040.380297:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764040.380300:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764040.380304:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764040.764267:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764040.764274:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764040.764276:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764040.764280:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764040.764285:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764040.764289:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764040.764292:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764040.764294:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764040.764297:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764041.020253:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316428019, at 4316428032
00000100:00000040:3.0:1787764041.020253:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316428019, at 4316428032
00000100:00000010:2.0:1787764041.020269:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00000100:00000010:3.0:1787764041.020273:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb72a0.
00000100:00000001:1.0:1787764041.404279:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764041.404282:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764041.404288:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764041.404290:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764041.404295:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764041.404295:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764041.404299:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764041.404304:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764041.404305:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764041.404311:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764041.404312:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764041.404315:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:1.0:1787764041.404317:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764041.404319:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764041.404325:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764041.404325:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764041.404327:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764041.404331:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764041.404332:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764041.404338:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764041.404339:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764041.404386:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764041.404392:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764041.404396:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764041.788239:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764041.788254:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764041.788256:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764041.788264:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764041.788268:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764041.788271:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764041.788274:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764041.788276:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764041.788279:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764042.428239:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764042.428246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764042.428253:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764042.428255:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764042.428258:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764042.428260:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764042.428263:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764042.428268:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764042.428273:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764042.428279:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764042.428284:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764042.428288:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764042.428292:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764042.428297:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764042.428302:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764042.428306:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764042.428312:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764042.428317:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764042.812241:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764042.812265:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764042.812268:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764042.812273:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764042.812277:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764042.812280:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764042.812283:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764042.812286:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764042.812289:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764043.452273:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764043.452276:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764043.452282:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764043.452284:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764043.452290:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764043.452291:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764043.452297:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764043.452301:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764043.452305:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764043.452308:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764043.452311:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764043.452314:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764043.452315:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764043.452320:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764043.452326:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764043.452376:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764043.452381:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764043.452384:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764043.836255:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764043.836264:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764043.836268:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764043.836274:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764043.836280:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764043.836285:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764043.836288:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764043.836291:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764043.836295:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764044.476205:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764044.476214:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764044.476217:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764044.476223:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764044.476235:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764044.476238:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764044.476240:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764044.476245:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764044.476248:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764044.476267:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764044.476272:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764044.476275:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764044.476277:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764044.476280:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764044.476281:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764044.476283:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764044.476285:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764044.476287:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764044.476288:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764044.476291:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764044.476292:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764044.476328:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764044.476331:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764044.476333:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787764044.860242:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764044.860251:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787764044.860254:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764044.860258:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764044.860262:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764044.860268:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21759
00000100:00000001:3.0:1787764044.860273:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000400:00000200:2.0:1787764044.860277:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764044.860278:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764044.860281:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764044.860284:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000400:00000200:2.0:1787764044.860285:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787764044.860289:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:2.0:1787764044.860290:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787764044.860293:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764044.860295:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764044.860297:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764044.860299:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764044.860300:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764044.860304:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764044.860305:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764044.860310:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764044.860314:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764044.860315:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787764044.860317:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764044.860320:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787764044.860323:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787764044.860325:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787764044.860327:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764044.860329:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764044.860331:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764044.860332:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764044.860334:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787764044.860336:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764044.860338:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764044.860340:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764044.860342:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764044.860344:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764044.860346:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764044.860347:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764044.860356:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000020:00000040:2.0:1787764044.860358:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787764044.860362:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764044.860363:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764044.860365:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764044.860367:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764044.860370:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764044.860376:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5b9800.
00000100:00000001:2.0:1787764044.860381:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764044.860389:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832685440/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764044.860396:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764044.860399:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764044.860403:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832685440/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764044.860423:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764044.860425:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764044.860428:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787764044.860429:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764044.860430:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764044.860432:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764044.860434:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21759
00000400:00000200:2.0:1787764044.860436:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764044.860438:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764044.860439:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764044.860441:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764044.860443:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764044.860445:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764044.860446:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764044.860448:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764044.860449:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:2.0:1787764044.860451:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764044.860452:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:2.0:1787764044.860455:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:2.0:1787764044.860456:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:0.0:1787764044.860460:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1787764044.860460:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764044.860462:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764044.860466:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764044.860467:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764044.860468:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:0.0:1787764044.860471:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:2.0:1787764044.860472:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:2.0:1787764044.860476:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764044.860478:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764044.860480:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:0.0:1787764044.860481:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832685440/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764044.860484:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764044.860490:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764044.860494:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764044.860497:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764044.860499:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:2.0:1787764044.860499:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:0.0:1787764044.860505:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:2.0:1787764044.860505:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764044.860507:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764044.860509:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764044.860511:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764044.860514:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832685440:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787764044.860521:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000010:2.0:1787764044.860523:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
02000000:00000001:0.0:1787764044.860525:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:2.0:1787764044.860528:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787764044.860529:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764044.860535:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764044.860536:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1787764044.860539:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764044.860541:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764044.860542:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:0.0:1787764044.860543:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9014400.
02000000:00000001:2.0:1787764044.860544:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764044.860549:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764044.860552:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bb800.
00000400:00000200:0.0:1787764044.860557:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764044.860561:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:0.0:1787764044.860562:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e528.
00000400:00000010:0.0:1787764044.860571:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669c38.
00000100:00100000:2.0:1787764044.860574:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832685568/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764044.860585:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:0.0:1787764044.860587:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832685440, portal 25
00000100:00000001:2.0:1787764044.860588:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764044.860593:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787764044.860598:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:2.0:1787764044.860598:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832685568/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764044.860610:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832685440/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.860634:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1787764044.860638:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764044.860639:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00080000:2.0:1787764044.860644:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:0.0:1787764044.860645:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00080000:2.0:1787764044.860650:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000200:0.0:1787764044.860652:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832685440, offset 0
00000100:00000001:2.0:1787764044.860653:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764044.860656:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:0.0:1787764044.860662:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787764044.860669:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:0.0:1787764044.860670:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:2.0:1787764044.860676:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21759
00000400:00000200:0.0:1787764044.860679:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787764044.860686:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764044.860687:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764044.860691:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764044.860696:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:0.0:1787764044.860697:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787764044.860703:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00080000:2.0:1787764044.860703:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:0.0:1787764044.860711:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58590:58590:256:4294967295] 192.168.203.110@tcp LPNI seq info [58590:58590:8:4294967295]
00000100:00000040:2.0:1787764044.860713:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:0.0:1787764044.860717:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764044.860718:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764044.860722:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764044.860725:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:0.0:1787764044.860730:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000020:00000001:2.0:1787764044.860730:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764044.860731:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787764044.860735:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764044.860739:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000200:0.0:1787764044.860742:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000020:00000001:2.0:1787764044.860743:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:2.0:1787764044.860748:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000010:0.0:1787764044.860750:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062200.
00000100:00000040:2.0:1787764044.860751:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764044.860754:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764044.860757:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000200:0.0:1787764044.860762:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787764044.860762:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:0.0:1787764044.860768:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000020:00000040:2.0:1787764044.860768:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:2.0:1787764044.860772:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764044.860776:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:0.1:1787764044.860777:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:2.0:1787764044.860780:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:0.1:1787764044.860782:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
02000000:00000001:2.0:1787764044.860785:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000100:0.0:1787764044.860789:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
02000000:00000001:2.0:1787764044.860790:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1787764044.860792:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062200.
00000100:00000001:2.0:1787764044.860794:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764044.860797:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:0.0:1787764044.860798:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000008:00000001:2.0:1787764044.860799:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764044.860802:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787764044.860804:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000010:2.0:1787764044.860808:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000400:00000200:0.0:1787764044.860809:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787764044.860814:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000020:00000040:2.0:1787764044.860814:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764044.860820:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787764044.860821:0:257575:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:2.0:1787764044.860822:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764044.860825:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:0.0:1787764044.860827:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832685440/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764044.860827:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764044.860830:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764044.860836:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5b8c00.
00000100:00000001:0.0:1787764044.860840:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787764044.860841:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764044.860846:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832685440/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764069 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764044.860849:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832685696/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764044.860863:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764044.860864:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764044.860868:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764044.860869:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:2.0:1787764044.860873:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832685696/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.860880:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764044.860880:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764044.860887:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:0.0:1787764044.860891:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:2.0:1787764044.860892:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:0.0:1787764044.860897:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316433139
00000100:00080000:2.0:1787764044.860897:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764044.860904:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787764044.860904:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764044.860906:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764044.860907:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000010:0.0:1787764044.860908:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c720.
00000100:00080000:2.0:1787764044.860912:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21759
00000100:00000001:0.0:1787764044.860931:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000400:00000200:2.0:1787764044.860933:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764044.860940:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316433139
00000400:00000200:2.0:1787764044.860940:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764044.860943:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000010:0.0:1787764044.860945:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9751d940c8a0.
00000100:00000040:2.0:1787764044.860949:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764044.860950:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787764044.860954:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:0.0:1787764044.860955:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1787764044.860958:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764044.860960:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764044.860962:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:0.0:1787764044.860964:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000020:00000001:2.0:1787764044.860965:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764044.860967:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787764044.860969:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00100000:0.0:1787764044.860971:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764044/real 1787764044]  req@ffff9751f86583c0 x1874599832685440/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787764044.860972:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787764044.860977:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:2.0:1787764044.860982:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:2.0:1787764044.860990:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000001:0.0:1787764044.860991:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:2.0:1787764044.860992:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000001:0.0:1787764044.860995:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:2.0:1787764044.860996:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000010:0.0:1787764044.860999:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e528.
00000100:00000001:2.0:1787764044.860999:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764044.861004:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000400:00000200:0.0:1787764044.861005:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669c38
00000100:00080000:2.0:1787764044.861007:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000010:0.0:1787764044.861009:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669c38.
00000100:00000001:2.0:1787764044.861010:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764044.861013:0:257575:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:2.0:1787764044.861013:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764044.861018:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:0.0:1787764044.861019:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832685440/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764044.861023:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764044.861028:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764044.861031:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000200:0.0:1787764044.861035:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832685440/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:2.0:1787764044.861036:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764044.861040:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764044.861046:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000100:00000001:0.0:1787764044.861047:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000020:00000040:2.0:1787764044.861050:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:0.0:1787764044.861051:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764044.861055:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:2.0:1787764044.861055:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:0.0:1787764044.861058:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787764044.861062:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832685440/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764044.861065:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764044.861068:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764044.861069:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764044.861071:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:0.0:1787764044.861074:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832685440/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764044.861075:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764044.861082:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bbc00.
00000100:00000001:2.0:1787764044.861087:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764044.861088:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787764044.861094:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764044.861097:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832685824/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764044.861098:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787764044.861102:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764044.861107:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764044.861111:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764044.861112:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764044.861114:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764044.861119:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000040:2.0:1787764044.861120:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832685824/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787764044.861130:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764044.861132:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764044.861155:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21769)
00000100:00000001:0.0:1787764044.861164:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787764044.861171:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832685440/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787764044.861177:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832685440:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787764044.861181:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764044.861183:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787764044.861185:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832685440/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861190:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787764044.861192:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787764044.861194:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9014400.
02000000:00000001:0.0:1787764044.861196:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787764044.861197:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787764044.861199:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:0.0:1787764044.861200:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787764044.861202:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:0.0:1787764044.861205:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787764044.861207:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787764044.861208:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:0.0:1787764044.861210:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787764044.861211:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764044.861212:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764044.861218:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764044.861219:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764044.861221:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787764044.861222:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764044.861223:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787764044.861225:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832685568/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861230:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764044.861231:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764044.861233:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787764044.861235:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764044.861237:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764044.861238:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832685568:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787764044.861241:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787764044.861243:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787764044.861244:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764044.861245:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787764044.861247:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9014400.
02000000:00000001:0.0:1787764044.861248:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787764044.861251:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787764044.861253:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e528.
00000400:00000010:0.0:1787764044.861256:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669c38.
00000100:00000200:0.0:1787764044.861258:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832685568, portal 10
00000100:00000001:0.0:1787764044.861260:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787764044.861261:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:0.0:1787764044.861265:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832685568/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861270:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787764044.861271:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787764044.861273:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787764044.861275:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832685568, offset 0
00000400:00000200:0.0:1787764044.861277:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787764044.861280:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764044.861283:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787764044.861287:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787764044.861290:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787764044.861293:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787764044.861295:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787764044.861298:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58591:58591:256:4294967295] 192.168.203.110@tcp LPNI seq info [58591:58591:8:4294967295]
00000400:00000200:0.0:1787764044.861303:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764044.861310:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787764044.861315:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787764044.861320:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062200.
00000800:00000200:0.0:1787764044.861325:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787764044.861328:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787764044.861333:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787764044.861336:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1787764044.861340:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787764044.861342:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062200.
00000400:00000200:0.0:1787764044.861344:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787764044.861346:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787764044.861349:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787764044.861350:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787764044.861353:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787764044.861357:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832685568/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861366:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787764044.861369:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832685568/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764069 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861383:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764044.861385:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787764044.861389:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764044.861393:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:0.0:1787764044.861397:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764044.861399:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316431904
00000100:00000001:0.0:1787764044.861401:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316433139
00000100:00000001:0.0:1787764044.861403:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316431904
00000100:00000001:0.0:1787764044.861404:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316433139
00000100:00000001:0.0:1787764044.861405:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764044.861407:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764044.861408:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787764044.861411:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764044/real 1787764044]  req@ffff9751f8658000 x1874599832685568/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787764044.861416:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787764044.861417:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787764044.861419:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e528.
00000400:00000200:0.0:1787764044.861421:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669c38
00000400:00000010:0.0:1787764044.861422:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669c38.
00000100:00000001:0.0:1787764044.861423:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787764044.861426:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832685568/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787764044.861431:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832685568/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861435:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787764044.861436:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764044.861437:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787764044.861438:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787764044.861440:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832685568/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861446:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787764044.861449:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832685568/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861453:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787764044.861454:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764044.861455:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787764044.861456:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764044.861457:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787764044.861467:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787764044.861468:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787764044.861470:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787764044.861472:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787764044.861474:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832685568/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787764044.861479:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832685568:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787764044.861481:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787764044.861484:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832685696/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861488:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764044.861489:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764044.861490:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787764044.861492:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764044.861493:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764044.861495:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832685696:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787764044.861497:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787764044.861498:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787764044.861500:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764044.861501:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787764044.861504:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9016000.
02000000:00000001:0.0:1787764044.861505:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787764044.861507:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787764044.861509:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e528.
00000400:00000010:0.0:1787764044.861511:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669c38.
00000100:00000200:0.0:1787764044.861513:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832685696, portal 10
00000100:00000001:0.0:1787764044.861514:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787764044.861515:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:0.0:1787764044.861519:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832685696/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861524:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787764044.861526:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787764044.861527:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787764044.861529:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832685696, offset 0
00000400:00000200:0.0:1787764044.861531:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787764044.861534:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764044.861536:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787764044.861538:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787764044.861540:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787764044.861542:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787764044.861543:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787764044.861545:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58592:58592:256:4294967295] 192.168.203.110@tcp LPNI seq info [58592:58592:8:4294967295]
00000400:00000200:0.0:1787764044.861548:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764044.861552:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787764044.861555:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787764044.861557:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062200.
00000800:00000200:0.0:1787764044.861560:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787764044.861562:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787764044.861564:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787764044.861566:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1787764044.861569:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787764044.861571:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062200.
00000400:00000200:0.0:1787764044.861573:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787764044.861575:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787764044.861577:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787764044.861578:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787764044.861580:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787764044.861583:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832685696/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861591:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787764044.861594:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832685696/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764069 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861601:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764044.861602:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787764044.861606:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764044.861609:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:0.0:1787764044.861612:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764044.861614:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316431892
00000100:00000001:0.0:1787764044.861617:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316433139
00000100:00000001:0.0:1787764044.861619:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316431892
00000100:00000001:0.0:1787764044.861622:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316433139
00000100:00000001:0.0:1787764044.861624:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764044.861627:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764044.861629:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787764044.861633:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764044/real 1787764044]  req@ffff9751f8659a40 x1874599832685696/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787764044.861648:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787764044.861649:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787764044.861650:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e528.
00000400:00000200:0.0:1787764044.861655:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669c38
00000400:00000010:0.0:1787764044.861657:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669c38.
00000100:00000001:0.0:1787764044.861659:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787764044.861661:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832685696/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787764044.861666:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832685696/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861671:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787764044.861672:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764044.861673:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787764044.861674:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787764044.861676:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832685696/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861680:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787764044.861683:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832685696/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861687:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787764044.861689:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764044.861689:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787764044.861690:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764044.861691:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787764044.861692:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787764044.861694:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787764044.861696:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787764044.861697:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787764044.861699:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832685696/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787764044.861704:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832685696:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787764044.861706:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787764044.861713:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832685824/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861717:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764044.861718:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764044.861720:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787764044.861721:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764044.861722:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764044.861724:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832685824:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787764044.861726:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787764044.861727:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787764044.861728:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764044.861730:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:0.0:1787764044.861732:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750e9015400.
02000000:00000001:0.0:1787764044.861734:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787764044.861736:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:0.0:1787764044.861738:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750ce70e528.
00000400:00000010:0.0:1787764044.861740:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669c38.
00000100:00000200:0.0:1787764044.861742:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832685824, portal 4
00000100:00000001:0.0:1787764044.861743:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787764044.861744:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:0.0:1787764044.861748:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832685824/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861752:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:0.0:1787764044.861753:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:0.0:1787764044.861755:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f4669770.
00000100:00000200:0.0:1787764044.861756:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832685824, offset 0
00000400:00000200:0.0:1787764044.861759:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787764044.861761:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764044.861764:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787764044.861766:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787764044.861768:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:0.0:1787764044.861769:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1787764044.861771:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787764044.861773:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58593:58593:256:4294967295] 192.168.203.110@tcp LPNI seq info [58593:58593:8:4294967295]
00000400:00000200:0.0:1787764044.861776:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764044.861780:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1787764044.861783:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787764044.861785:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f6062200.
00000800:00000200:0.0:1787764044.861788:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.0:1787764044.861790:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:0.1:1787764044.861793:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:0.1:1787764044.861795:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:0.0:1787764044.861797:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:0.0:1787764044.861798:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f6062200.
00000400:00000200:0.0:1787764044.861810:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:0.0:1787764044.861812:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:0.0:1787764044.861814:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669770
00000400:00000010:0.0:1787764044.861815:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669770.
00000100:00000001:0.0:1787764044.861817:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787764044.861819:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832685824/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764069 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861824:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787764044.861826:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832685824/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764069 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861830:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764044.861831:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787764044.861833:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764044.861835:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:0.0:1787764044.861837:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764044.861838:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316431909
00000100:00000001:0.0:1787764044.861839:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316433139
00000100:00000001:0.0:1787764044.861840:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316431909
00000100:00000001:0.0:1787764044.861842:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316433139
00000100:00000001:0.0:1787764044.861844:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764044.861845:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764044.861847:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:0.0:1787764044.861849:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764044/real 1787764044]  req@ffff9751f865ad00 x1874599832685824/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:0.0:1787764044.861854:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:0.0:1787764044.861855:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:0.0:1787764044.861856:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750ce70e528.
00000400:00000200:0.0:1787764044.861857:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f4669c38
00000400:00000010:0.0:1787764044.861859:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f4669c38.
00000100:00000001:0.0:1787764044.861861:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:0.0:1787764044.861864:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832685824/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787764044.861874:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832685824/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861880:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787764044.861882:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764044.861884:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787764044.861885:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787764044.861889:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832685824/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861897:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787764044.861901:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832685824/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861909:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787764044.861911:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764044.861912:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787764044.861914:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764044.861915:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:0.0:1787764044.861928:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787764044.861931:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:0.0:1787764044.861934:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787764044.861936:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:0.0:1787764044.861939:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832685824/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787764044.861944:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832685824:192.168.203.110@tcp:8:kworker.0
00000100:00000001:0.0:1787764044.861946:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764044.861948:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787764044.861949:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832685568/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861954:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787764044.861955:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787764044.861956:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9014400.
02000000:00000001:0.0:1787764044.861958:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787764044.861959:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787764044.861960:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787764044.861962:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787764044.861963:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:0.0:1787764044.861965:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787764044.861966:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787764044.861967:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:0.0:1787764044.861968:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787764044.861969:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764044.861971:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787764044.861973:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832685696/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861976:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787764044.861977:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787764044.861978:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9016000.
02000000:00000001:0.0:1787764044.861981:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787764044.861982:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787764044.861982:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:0.0:1787764044.861984:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787764044.861985:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:0.0:1787764044.861986:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787764044.861987:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787764044.861988:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:0.0:1787764044.861990:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787764044.861990:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764044.861991:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787764044.861993:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832685824/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764069 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764044.861997:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787764044.861997:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:0.0:1787764044.861998:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750e9015400.
02000000:00000001:0.0:1787764044.862000:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:0.0:1787764044.862000:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:0.0:1787764044.862001:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:0.0:1787764044.862002:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:0.0:1787764044.862003:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:0.0:1787764044.862004:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787764044.862005:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:0.0:1787764044.862005:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:0.0:1787764044.862007:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:0.0:1787764044.862007:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764044.862009:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764044.862011:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764044.862012:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764044.862013:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787764044.862014:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764045.500240:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764045.500243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764045.500248:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764045.500251:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764045.500254:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764045.500255:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764045.500258:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764045.500261:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764045.500264:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764045.500267:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764045.500268:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764045.500273:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764045.500276:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764045.500279:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764045.500282:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764045.500304:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764045.500308:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764045.500311:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764045.884286:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764045.884295:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764045.884299:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764045.884305:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764045.884310:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764045.884316:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764045.884319:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764045.884322:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764045.884326:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764046.140235:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316433139, at 4316433152
00000100:00000010:3.0:1787764046.140250:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9751d940c8a0.
00000100:00000040:2.0:1787764046.140283:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316433139, at 4316433152
00000100:00000010:2.0:1787764046.140293:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9751d940c720.
00000100:00000001:1.0:1787764046.524222:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764046.524222:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764046.524229:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764046.524230:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764046.524234:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764046.524236:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764046.524238:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764046.524239:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764046.524244:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764046.524245:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764046.524248:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764046.524248:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764046.524252:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764046.524254:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764046.524256:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764046.524298:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764046.524303:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764046.524307:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764046.908231:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764046.908238:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764046.908240:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764046.908244:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764046.908248:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764046.908250:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764046.908253:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764046.908255:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764046.908257:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764047.548214:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764047.548219:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764047.548222:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764047.548223:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764047.548225:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764047.548229:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764047.548231:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764047.548234:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:1.0:1787764047.548235:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764047.548237:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:1.0:1787764047.548240:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764047.548240:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764047.548246:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:1.0:1787764047.548247:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764047.548249:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764047.548252:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764047.548253:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764047.548259:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764047.548260:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764047.548268:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764047.548270:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764047.548308:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764047.548311:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764047.548314:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764047.932214:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764047.932220:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764047.932223:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764047.932229:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764047.932234:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764047.932238:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764047.932241:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764047.932244:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764047.932247:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764048.572191:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764048.572198:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764048.572200:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764048.572205:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764048.572208:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764048.572211:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764048.572212:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764048.572215:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764048.572216:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764048.572219:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764048.572222:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764048.572223:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764048.572224:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764048.572229:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764048.572231:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764048.572272:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764048.572277:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764048.572281:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764048.956222:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764048.956230:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764048.956233:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764048.956240:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764048.956244:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764048.956248:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764048.956250:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764048.956253:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764048.956256:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764049.596246:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764049.596266:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764049.596270:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764049.596273:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764049.596273:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764049.596280:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764049.596284:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764049.596284:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764049.596289:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764049.596290:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764049.596295:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764049.596297:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764049.596298:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764049.596300:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764049.596303:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764049.596332:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764049.596335:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764049.596338:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764049.980218:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764049.980225:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764049.980227:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764049.980232:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764049.980236:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764049.980239:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764049.980242:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764049.980244:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764049.980247:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764049.980279:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764049.980283:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764049.980285:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764049.980287:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764049.980290:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764049.980296:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21764
00000400:00000200:3.0:1787764049.980302:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764049.980307:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764049.980310:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764049.980313:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764049.980315:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764049.980317:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764049.980318:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764049.980321:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764049.980322:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787764049.980324:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764049.980327:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787764049.980328:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787764049.980330:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787764049.980332:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764049.980333:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764049.980335:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764049.980336:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764049.980338:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787764049.980340:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764049.980342:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764049.980344:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764049.980347:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764049.980349:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764049.980350:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764049.980352:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764049.980359:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bde00.
00000020:00000040:3.0:1787764049.980362:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787764049.980364:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764049.980366:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764049.980368:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764049.980370:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764049.980373:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764049.980379:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613800.
00000100:00000001:3.0:1787764049.980384:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764049.980391:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bde00 x1874599832685952/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764049.980399:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764049.980410:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764049.980415:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000941163c2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bde00 x1874599832685952/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764049.980432:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764049.980434:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764049.980436:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787764049.980437:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764049.980438:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764049.980440:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764049.980442:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21764
00000400:00000200:3.0:1787764049.980445:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764049.980449:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764049.980451:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764049.980453:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764049.980455:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764049.980456:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1787764049.980457:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764049.980461:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764049.980462:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1787764049.980463:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1787764049.980465:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:3.0:1787764049.980466:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:3.0:1787764049.980469:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764049.980471:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000040:2.0:1787764049.980475:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bde00 x1874599832685952/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787764049.980481:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:3.0:1787764049.980486:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787764049.980488:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764049.980492:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764049.980493:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764049.980498:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1787764049.980499:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:3.0:1787764049.980503:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764049.980504:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764049.980505:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:2.0:1787764049.980511:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1787764049.980512:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:3.0:1787764049.980516:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764049.980517:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764049.980518:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764049.980523:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:2.0:1787764049.980525:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832685952:192.168.203.110@tcp:250:kworker.0
02000000:00000001:3.0:1787764049.980526:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764049.980530:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764049.980534:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:3.0:1787764049.980534:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1787764049.980538:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00000001:3.0:1787764049.980540:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764049.980541:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764049.980545:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1787764049.980545:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764049.980552:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
00000100:00000010:3.0:1787764049.980553:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bfc00.
02000000:00000001:2.0:1787764049.980555:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1787764049.980560:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000200:2.0:1787764049.980563:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:3.0:1787764049.980568:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:2.0:1787764049.980570:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
02000000:00000001:3.0:1787764049.980573:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764049.980577:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000001:3.0:1787764049.980577:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764049.980582:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:2.0:1787764049.980586:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832685952, portal 25
02000000:00000001:3.0:1787764049.980586:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764049.980590:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764049.980593:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049853440 : -115096659698176 : ffff9751f87bde00)
02000000:00000010:3.0:1787764049.980595:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613c00.
00000100:00000040:2.0:1787764049.980600:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bde00 x1874599832685952/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764049.980601:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764049.980610:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00100000:3.0:1787764049.980612:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bfc00 x1874599832686080/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:2.0:1787764049.980615:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764049.980620:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764049.980625:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832685952, offset 0
00000100:00000001:3.0:1787764049.980625:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764049.980630:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764049.980635:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000040:3.0:1787764049.980636:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003c1f6b31] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bfc00 x1874599832686080/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764049.980645:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764049.980646:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:3.0:1787764049.980652:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787764049.980657:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00080000:3.0:1787764049.980657:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1787764049.980660:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764049.980663:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1787764049.980664:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764049.980669:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764049.980672:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764049.980677:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:3.0:1787764049.980677:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:2.0:1787764049.980682:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58594:58594:256:4294967295] 192.168.203.110@tcp LPNI seq info [58594:58594:8:4294967295]
00000100:00080000:3.0:1787764049.980683:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21764
00000400:00000200:2.0:1787764049.980688:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764049.980696:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764049.980703:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764049.980704:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000010:2.0:1787764049.980710:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000400:00000200:3.0:1787764049.980714:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764049.980718:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000200:2.0:1787764049.980719:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764049.980725:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000100:00000040:3.0:1787764049.980727:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:2.1:1787764049.980733:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787764049.980734:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.1:1787764049.980739:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000100:00000040:3.0:1787764049.980742:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000100:2.0:1787764049.980747:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787764049.980748:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000010:2.0:1787764049.980750:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000020:00000001:3.0:1787764049.980752:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764049.980755:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:3.0:1787764049.980756:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:2.0:1787764049.980757:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:3.0:1787764049.980759:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000400:00000200:2.0:1787764049.980764:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:3.0:1787764049.980764:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:3.0:1787764049.980769:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:2.0:1787764049.980773:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000100:00000040:3.0:1787764049.980775:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000010:2.0:1787764049.980776:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:3.0:1787764049.980779:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764049.980782:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:3.0:1787764049.980784:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000200:2.0:1787764049.980789:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bde00 x1874599832685952/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764049.980789:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764049.980795:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00000001:2.0:1787764049.980804:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00080000:3.0:1787764049.980808:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000040:2.0:1787764049.980809:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bde00 x1874599832685952/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764074 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764049.980813:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764049.980817:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787764049.980822:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764049.980822:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764049.980828:0:257575:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:3.0:1787764049.980829:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764049.980831:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764049.980834:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1787764049.980834:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787764049.980836:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764049.980840:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764049.980841:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764049.980846:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316438259
00000100:00000010:3.0:1787764049.980848:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc3c0.
00000100:00000010:2.0:1787764049.980852:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000020:00000040:3.0:1787764049.980852:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787764049.980858:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764049.980859:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764049.980861:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787764049.980861:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764049.980862:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764049.980866:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764049.980870:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316438259
02000000:00000010:3.0:1787764049.980871:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611800.
00000100:00000010:2.0:1787764049.980877:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7180.
00000100:00000001:3.0:1787764049.980878:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764049.980897:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc3c0 x1874599832686208/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764049.980901:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787764049.980907:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764049.980907:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764049.980911:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764049.980912:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764049.980915:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000040:3.0:1787764049.980917:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000060bf8470] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc3c0 x1874599832686208/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764049.980923:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764049/real 1787764049]  req@ffff9751f87bde00 x1874599832685952/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764049.980931:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764049.980936:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1787764049.980944:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:3.0:1787764049.980949:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000001:2.0:1787764049.980954:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00080000:3.0:1787764049.980955:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000001:2.0:1787764049.980957:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:3.0:1787764049.980958:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:2.0:1787764049.980961:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000100:00080000:3.0:1787764049.980961:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1787764049.980965:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000100:00000001:3.0:1787764049.980966:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:2.0:1787764049.980970:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00080000:3.0:1787764049.980972:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21764
00000100:00000001:2.0:1787764049.980974:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764049.980980:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bde00 x1874599832685952/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764049.980980:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764049.980987:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764049.980992:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000200:2.0:1787764049.980997:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bde00 x1874599832685952/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787764049.980999:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764049.981005:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764049.981010:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764049.981012:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764049.981013:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764049.981017:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:3.0:1787764049.981017:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764049.981018:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:3.0:1787764049.981021:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1787764049.981024:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000040:3.0:1787764049.981025:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000001:2.0:1787764049.981029:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:3.0:1787764049.981030:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:3.0:1787764049.981036:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:3.0:1787764049.981040:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00100000:2.0:1787764049.981042:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bde00 x1874599832685952/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764049.981042:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764049.981047:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764049.981052:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764049.981057:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:3.0:1787764049.981063:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764049.981067:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764049.981067:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764049.981073:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764049.981076:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764049.981080:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764049.981081:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bde00 x1874599832685952/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764049.981082:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787764049.981086:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1787764049.981091:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764049.981095:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764049.981101:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000010:3.0:1787764049.981101:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc780.
00000020:00000040:3.0:1787764049.981105:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:2.0:1787764049.981107:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787764049.981109:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1787764049.981110:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:3.0:1787764049.981113:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764049.981114:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764049.981119:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1787764049.981120:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
02000000:00000001:3.0:1787764049.981122:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787764049.981123:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:3.0:1787764049.981124:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764049.981127:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000010:3.0:1787764049.981130:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610400.
00000100:00080000:2.0:1787764049.981131:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764049.981152:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764049.981156:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bde00 x1874599832685952/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764049.981158:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764049.981168:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc780 x1874599832686336/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:2.0:1787764049.981170:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832685952:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764049.981182:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764049.981183:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764049.981187:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1787764049.981188:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764049.981200:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7d56184] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc780 x1874599832686336/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787764049.981201:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bde00 x1874599832685952/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764049.981218:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764049.981225:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21774)
00000100:00000001:2.0:1787764049.981226:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764049.981230:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764049.981233:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764049.981236:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764049.981238:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764049.981239:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764049.981241:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764049.981243:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613800.
02000000:00000001:2.0:1787764049.981247:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764049.981248:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764049.981249:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bde00.
00000100:00000001:2.0:1787764049.981252:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764049.981253:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764049.981255:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764049.981260:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764049.981261:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764049.981263:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764049.981264:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764049.981265:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764049.981269:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bfc00 x1874599832686080/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981273:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764049.981275:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764049.981276:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764049.981278:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764049.981279:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764049.981281:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832686080:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764049.981284:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764049.981285:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764049.981287:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764049.981288:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764049.981290:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764049.981292:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764049.981295:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764049.981297:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764049.981299:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764049.981302:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832686080, portal 10
00000100:00000001:2.0:1787764049.981303:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764049.981304:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049861120 : -115096659690496 : ffff9751f87bfc00)
00000100:00000040:2.0:1787764049.981309:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bfc00 x1874599832686080/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981313:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764049.981315:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764049.981316:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764049.981318:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832686080, offset 0
00000400:00000200:2.0:1787764049.981320:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764049.981323:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764049.981326:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764049.981328:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764049.981330:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764049.981332:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764049.981334:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764049.981336:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58595:58595:256:4294967295] 192.168.203.110@tcp LPNI seq info [58595:58595:8:4294967295]
00000400:00000200:2.0:1787764049.981338:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764049.981343:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764049.981346:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764049.981348:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764049.981351:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764049.981353:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787764049.981356:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764049.981358:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:2.0:1787764049.981361:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764049.981362:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764049.981364:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764049.981367:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764049.981369:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764049.981370:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764049.981372:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764049.981374:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bfc00 x1874599832686080/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981379:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764049.981381:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bfc00 x1874599832686080/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764074 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981386:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764049.981387:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764049.981389:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764049.981392:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764049.981394:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764049.981395:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316437024
00000100:00000001:2.0:1787764049.981397:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316438259
00000100:00000001:2.0:1787764049.981399:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316437024
00000100:00000001:2.0:1787764049.981400:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316438259
00000100:00000001:2.0:1787764049.981402:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764049.981403:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764049.981404:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764049.981406:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764049/real 1787764049]  req@ffff9751f87bfc00 x1874599832686080/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764049.981411:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764049.981412:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764049.981413:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764049.981415:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764049.981416:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764049.981417:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764049.981419:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bfc00 x1874599832686080/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764049.981424:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bfc00 x1874599832686080/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981428:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764049.981429:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764049.981430:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764049.981431:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764049.981433:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bfc00 x1874599832686080/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981437:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764049.981439:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bfc00 x1874599832686080/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981443:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764049.981445:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764049.981446:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764049.981447:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764049.981448:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764049.981458:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764049.981460:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764049.981462:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764049.981463:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764049.981466:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bfc00 x1874599832686080/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764049.981471:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832686080:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764049.981473:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764049.981476:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc3c0 x1874599832686208/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981480:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764049.981481:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764049.981482:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764049.981483:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764049.981485:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764049.981486:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832686208:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764049.981488:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764049.981489:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764049.981490:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764049.981492:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764049.981494:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764049.981495:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764049.981497:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764049.981499:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764049.981500:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764049.981502:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832686208, portal 10
00000100:00000001:2.0:1787764049.981503:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764049.981504:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049846720 : -115096659704896 : ffff9751f87bc3c0)
00000100:00000040:2.0:1787764049.981509:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc3c0 x1874599832686208/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981513:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764049.981515:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764049.981516:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764049.981518:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832686208, offset 0
00000400:00000200:2.0:1787764049.981520:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764049.981522:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764049.981524:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764049.981525:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764049.981528:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764049.981529:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764049.981531:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764049.981533:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58596:58596:256:4294967295] 192.168.203.110@tcp LPNI seq info [58596:58596:8:4294967295]
00000400:00000200:2.0:1787764049.981535:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764049.981539:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764049.981542:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764049.981543:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764049.981546:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764049.981547:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787764049.981550:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764049.981552:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:2.0:1787764049.981554:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764049.981556:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764049.981557:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764049.981560:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764049.981561:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764049.981563:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764049.981564:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764049.981566:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc3c0 x1874599832686208/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981571:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764049.981573:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc3c0 x1874599832686208/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764074 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981577:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764049.981578:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764049.981580:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764049.981581:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764049.981583:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764049.981584:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316437012
00000100:00000001:2.0:1787764049.981585:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316438259
00000100:00000001:2.0:1787764049.981587:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316437012
00000100:00000001:2.0:1787764049.981589:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316438259
00000100:00000001:2.0:1787764049.981590:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764049.981591:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764049.981592:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764049.981595:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764049/real 1787764049]  req@ffff9751f87bc3c0 x1874599832686208/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764049.981599:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764049.981600:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764049.981601:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764049.981602:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764049.981603:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764049.981604:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764049.981606:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc3c0 x1874599832686208/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764049.981611:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc3c0 x1874599832686208/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981616:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764049.981616:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764049.981617:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764049.981618:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764049.981620:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc3c0 x1874599832686208/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981624:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764049.981626:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc3c0 x1874599832686208/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981630:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764049.981632:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764049.981633:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764049.981634:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764049.981635:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764049.981636:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764049.981638:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764049.981640:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764049.981642:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764049.981644:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc3c0 x1874599832686208/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764049.981648:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832686208:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764049.981650:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764049.981652:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc780 x1874599832686336/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981656:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764049.981657:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764049.981658:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764049.981659:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764049.981660:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764049.981662:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832686336:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764049.981664:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764049.981665:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764049.981666:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764049.981667:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764049.981669:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764049.981670:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764049.981672:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764049.981674:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764049.981676:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764049.981678:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832686336, portal 4
00000100:00000001:2.0:1787764049.981679:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764049.981680:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049847680 : -115096659703936 : ffff9751f87bc780)
00000100:00000040:2.0:1787764049.981684:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc780 x1874599832686336/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981689:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764049.981691:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764049.981692:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764049.981693:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832686336, offset 0
00000400:00000200:2.0:1787764049.981695:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764049.981697:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764049.981699:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764049.981701:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764049.981702:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764049.981704:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764049.981705:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764049.981707:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58597:58597:256:4294967295] 192.168.203.110@tcp LPNI seq info [58597:58597:8:4294967295]
00000400:00000200:2.0:1787764049.981710:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764049.981714:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764049.981716:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764049.981718:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764049.981720:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764049.981721:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787764049.981724:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764049.981725:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:2.0:1787764049.981728:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764049.981729:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764049.981731:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764049.981733:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764049.981735:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764049.981736:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764049.981737:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764049.981740:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc780 x1874599832686336/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764074 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981744:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764049.981746:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc780 x1874599832686336/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764074 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981755:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764049.981756:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764049.981758:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764049.981760:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764049.981761:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764049.981763:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316437029
00000100:00000001:2.0:1787764049.981764:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316438259
00000100:00000001:2.0:1787764049.981766:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316437029
00000100:00000001:2.0:1787764049.981767:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316438259
00000100:00000001:2.0:1787764049.981768:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764049.981769:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764049.981770:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764049.981772:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764049/real 1787764049]  req@ffff9751f87bc780 x1874599832686336/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764049.981777:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764049.981777:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764049.981778:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764049.981780:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764049.981781:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764049.981782:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764049.981784:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc780 x1874599832686336/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764049.981789:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc780 x1874599832686336/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981793:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764049.981794:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764049.981795:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764049.981795:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764049.981797:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc780 x1874599832686336/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981802:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764049.981805:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc780 x1874599832686336/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981809:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764049.981811:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764049.981811:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764049.981812:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764049.981813:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764049.981814:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764049.981816:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764049.981817:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764049.981819:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764049.981821:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc780 x1874599832686336/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764049.981825:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832686336:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764049.981828:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764049.981829:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764049.981830:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bfc00 x1874599832686080/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981834:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764049.981835:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764049.981837:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764049.981838:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764049.981839:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764049.981840:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764049.981842:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764049.981843:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613c00.
02000000:00000001:2.0:1787764049.981844:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764049.981846:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764049.981847:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bfc00.
00000100:00000001:2.0:1787764049.981848:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764049.981849:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764049.981850:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764049.981852:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc3c0 x1874599832686208/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981856:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764049.981857:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764049.981858:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764049.981860:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764049.981860:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764049.981861:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764049.981862:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764049.981863:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611800.
02000000:00000001:2.0:1787764049.981864:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764049.981865:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764049.981866:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc3c0.
00000100:00000001:2.0:1787764049.981867:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764049.981868:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764049.981869:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764049.981870:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc780 x1874599832686336/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764074 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764049.981874:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764049.981875:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764049.981876:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764049.981877:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764049.981878:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764049.981878:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764049.981879:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764049.981880:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610400.
02000000:00000001:2.0:1787764049.981900:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764049.981901:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764049.981901:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc780.
00000100:00000001:2.0:1787764049.981903:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764049.981905:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764049.981906:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764049.981908:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764049.981909:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764049.981910:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764049.981912:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764050.620205:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764050.620212:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764050.620214:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764050.620215:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764050.620219:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764050.620225:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764050.620225:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764050.620227:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764050.620230:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764050.620232:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764050.620234:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764050.620235:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764050.620237:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764050.620240:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764050.620243:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764050.620271:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764050.620275:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764050.620278:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764051.004264:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764051.004273:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764051.004277:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764051.004283:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764051.004288:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764051.004292:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764051.004295:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764051.004298:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764051.004302:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764051.260237:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316438259, at 4316438272
00000100:00000010:3.0:1787764051.260252:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7180.
00000100:00000040:2.0:1787764051.260269:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316438259, at 4316438272
00000100:00000010:2.0:1787764051.260277:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00010000:00000001:3.0:1787764051.644253:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764051.644254:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764051.644261:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764051.644262:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764051.644264:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764051.644269:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764051.644271:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764051.644272:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764051.644277:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764051.644277:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764051.644283:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764051.644287:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764051.644287:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764051.644291:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764051.644294:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764051.644298:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764051.644302:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764051.644306:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764051.644308:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764051.644310:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764051.644314:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764051.644351:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764051.644354:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764051.644357:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764052.028244:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764052.028252:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764052.028254:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764052.028260:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764052.028264:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764052.028268:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764052.028270:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764052.028273:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764052.028276:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764052.668207:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764052.668207:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764052.668215:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764052.668216:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764052.668220:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764052.668222:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764052.668225:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764052.668225:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764052.668230:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764052.668232:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764052.668239:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764052.668242:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764052.668245:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764052.668248:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764052.668251:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764052.668288:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764052.668292:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764052.668295:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764053.052264:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764053.052274:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764053.052276:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764053.052281:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764053.052285:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764053.052289:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764053.052292:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764053.052294:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764053.052297:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764053.692213:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764053.692220:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764053.692221:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764053.692223:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764053.692225:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764053.692230:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764053.692231:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764053.692235:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764053.692235:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764053.692241:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764053.692242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764053.692244:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764053.692245:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764053.692247:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764053.692250:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764053.692278:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764053.692282:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764053.692286:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764054.076190:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764054.076196:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764054.076198:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764054.076203:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764054.076207:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764054.076210:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764054.076212:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764054.076215:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764054.076217:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764054.716190:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764054.716202:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764054.716205:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764054.716212:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764054.716218:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764054.716223:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764054.716226:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764054.716229:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764054.716234:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764054.716234:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764054.716241:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764054.716244:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764054.716249:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764054.716253:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764054.716256:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764054.716260:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764054.716264:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764054.716269:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764054.716271:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764054.716275:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764054.716276:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764054.716314:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764054.716320:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764054.716324:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764055.100264:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764055.100276:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764055.100279:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764055.100287:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764055.100292:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764055.100297:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764055.100300:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764055.100304:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764055.100309:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764055.100319:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764055.100326:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787764055.100330:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764055.100336:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764055.100342:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764055.100352:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21769
00000400:00000200:1.0:1787764055.100361:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764055.100371:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764055.100375:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764055.100380:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764055.100384:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764055.100388:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764055.100391:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764055.100397:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764055.100400:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787764055.100404:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764055.100408:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787764055.100410:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787764055.100413:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787764055.100416:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764055.100418:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764055.100421:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764055.100424:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764055.100428:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787764055.100432:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764055.100436:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764055.100441:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764055.100445:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764055.100449:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764055.100451:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764055.100454:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764055.100461:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7931680.
00000020:00000040:1.0:1787764055.100465:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787764055.100470:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764055.100472:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764055.100477:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764055.100481:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764055.100486:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764055.100496:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca400.
00000100:00000001:1.0:1787764055.100505:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764055.100519:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7931680 x1874599832686464/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764055.100533:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764055.100540:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764055.100546:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d4255d6] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7931680 x1874599832686464/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764055.100569:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764055.100572:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764055.100576:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787764055.100578:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764055.100581:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764055.100585:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764055.100588:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21769
00000400:00000200:1.0:1787764055.100593:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764055.100598:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764055.100601:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764055.100605:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764055.100607:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764055.100609:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764055.100611:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764055.100613:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764055.100614:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787764055.100615:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764055.100617:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787764055.100619:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787764055.100619:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:1.0:1787764055.100621:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787764055.100626:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764055.100629:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764055.100631:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1787764055.100632:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764055.100637:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764055.100637:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:1.0:1787764055.100643:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787764055.100648:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764055.100655:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764055.100655:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7931680 x1874599832686464/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787764055.100661:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764055.100670:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764055.100676:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:1.0:1787764055.100677:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764055.100682:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764055.100683:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1787764055.100692:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:1.0:1787764055.100693:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1787764055.100699:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:1.0:1787764055.100700:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764055.100707:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764055.100708:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764055.100717:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832686464:192.168.203.110@tcp:250:kworker.0
00000100:00000010:1.0:1787764055.100718:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930b40.
00000020:00000040:1.0:1787764055.100728:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000100:00000001:2.0:1787764055.100729:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764055.100736:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764055.100737:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764055.100739:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764055.100741:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764055.100744:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764055.100747:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:1.0:1787764055.100748:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764055.100752:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764055.100757:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8c00.
00000100:00000001:1.0:1787764055.100765:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764055.100772:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
00000100:00100000:1.0:1787764055.100775:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930b40 x1874599832686592/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1787764055.100778:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764055.100795:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764055.100801:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764055.100802:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764055.100810:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000100:00000040:1.0:1787764055.100814:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022d7f91a] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930b40 x1874599832686592/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1787764055.100821:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764055.100833:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832686464, portal 25
00000100:00000001:1.0:1787764055.100834:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764055.100838:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00080000:1.0:1787764055.100840:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764055.100842:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497727104 : -115097211824512 : ffff9751d7931680)
00000100:00080000:1.0:1787764055.100863:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787764055.100867:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764055.100869:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:2.0:1787764055.100872:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7931680 x1874599832686464/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764080 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764055.100875:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764055.100881:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21769
00000400:00000200:1.0:1787764055.100898:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764055.100902:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:1.0:1787764055.100908:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764055.100916:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764055.100916:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764055.100925:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000040:1.0:1787764055.100927:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764055.100933:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:2.0:1787764055.100934:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832686464, offset 0
00000100:00000040:1.0:1787764055.100939:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764055.100945:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764055.100953:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:2.0:1787764055.100954:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000020:00000040:1.0:1787764055.100956:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787764055.100961:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764055.100964:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000400:00000200:2.0:1787764055.100970:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:1.0:1787764055.100971:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787764055.100977:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:2.0:1787764055.100983:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000040:1.0:1787764055.100984:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764055.100992:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764055.100996:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764055.100998:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1787764055.100999:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764055.101005:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787764055.101013:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:2.0:1787764055.101014:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:1.0:1787764055.101017:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764055.101020:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1787764055.101024:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
02000000:00000001:1.0:1787764055.101025:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764055.101032:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764055.101036:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:2.0:1787764055.101037:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000008:00100000:1.0:1787764055.101039:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787764055.101043:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764055.101048:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764055.101053:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58598:58598:256:4294967295] 192.168.203.110@tcp LPNI seq info [58598:58598:8:4294967295]
00000100:00000010:1.0:1787764055.101055:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930000.
00000020:00000040:1.0:1787764055.101064:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764055.101069:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764055.101071:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764055.101074:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764055.101075:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764055.101080:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764055.101083:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764055.101088:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8800.
00000100:00000001:1.0:1787764055.101094:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764055.101102:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00100000:1.0:1787764055.101103:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930000 x1874599832686720/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764055.101126:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764055.101157:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764055.101168:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6d90274] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930000 x1874599832686720/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764055.101179:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764055.101182:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:2.0:1787764055.101185:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000040:1.0:1787764055.101187:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787764055.101196:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:2.0:1787764055.101197:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000100:00080000:1.0:1787764055.101204:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787764055.101208:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764055.101211:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:2.0:1787764055.101219:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787764055.101220:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764055.101226:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21769
00000800:00000200:2.0:1787764055.101229:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000400:00000200:1.0:1787764055.101233:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764055.101241:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764055.101252:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000200:2.1:1787764055.101253:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:1.0:1787764055.101259:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764055.101265:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.1:1787764055.101266:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000100:00000040:1.0:1787764055.101267:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764055.101272:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764055.101278:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764055.101279:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787764055.101284:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764055.101288:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000800:00000100:2.0:1787764055.101288:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000020:00000001:1.0:1787764055.101295:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787764055.101299:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000800:00000010:2.0:1787764055.101299:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000100:00000040:1.0:1787764055.101309:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764055.101312:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1787764055.101312:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:1.0:1787764055.101316:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764055.101320:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764055.101322:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:1.0:1787764055.101327:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:2.0:1787764055.101331:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:1.0:1787764055.101332:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764055.101336:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764055.101342:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764055.101347:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764055.101347:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000100:00000001:1.0:1787764055.101349:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764055.101355:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000400:00000010:2.0:1787764055.101355:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000008:00000001:1.0:1787764055.101364:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764055.101366:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764055.101369:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933480.
00000100:00000001:2.0:1787764055.101369:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000020:00000040:1.0:1787764055.101377:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787764055.101379:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764055.101381:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764055.101383:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:2.0:1787764055.101384:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7931680 x1874599832686464/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764080 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787764055.101385:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764055.101388:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764055.101393:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb800.
00000100:00000001:1.0:1787764055.101401:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764055.101411:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933480 x1874599832686848/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764055.101423:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764055.101423:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787764055.101429:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764055.101439:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3c2c33c] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933480 x1874599832686848/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787764055.101440:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7931680 x1874599832686464/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764080 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764055.101469:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764055.101478:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21779)
00000100:00000001:2.0:1787764055.101480:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764055.101485:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764055.101493:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764055.101499:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764055.101502:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316443380
00000100:00000010:2.0:1787764055.101507:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7720.
00000100:00000001:2.0:1787764055.101514:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787764055.101521:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316443380
00000100:00000010:2.0:1787764055.101524:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb70c0.
00000100:00000001:2.0:1787764055.101528:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787764055.101531:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764055.101535:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764055.101537:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764055.101543:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764055/real 1787764055]  req@ffff9751d7931680 x1874599832686464/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764055.101554:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764055.101556:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764055.101558:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764055.101561:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764055.101563:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764055.101566:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764055.101569:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7931680 x1874599832686464/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764055.101585:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7931680 x1874599832686464/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.101593:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764055.101595:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764055.101597:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764055.101599:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764055.101603:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7931680 x1874599832686464/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.101610:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764055.101616:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7931680 x1874599832686464/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.101623:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764055.101626:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764055.101627:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764055.101629:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764055.101631:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764055.101634:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764055.101638:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764055.101643:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764055.101645:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764055.101650:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7931680 x1874599832686464/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764055.101658:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832686464:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764055.101663:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764055.101666:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764055.101669:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7931680 x1874599832686464/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.101682:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764055.101685:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764055.101688:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764055.101691:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764055.101694:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764055.101696:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764055.101698:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764055.101701:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca400.
02000000:00000001:2.0:1787764055.101706:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764055.101707:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764055.101710:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7931680.
00000100:00000001:2.0:1787764055.101713:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764055.101714:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764055.101717:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764055.101723:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764055.101725:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764055.101728:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764055.101730:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764055.101731:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764055.101736:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930b40 x1874599832686592/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.101744:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764055.101745:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764055.101748:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764055.101750:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764055.101753:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764055.101756:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832686592:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764055.101760:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764055.101763:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764055.101765:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764055.101767:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764055.101770:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764055.101772:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764055.101776:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764055.101780:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764055.101784:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764055.101788:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832686592, portal 10
00000100:00000001:2.0:1787764055.101790:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764055.101791:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497724224 : -115097211827392 : ffff9751d7930b40)
00000100:00000040:2.0:1787764055.101799:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930b40 x1874599832686592/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764080 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.101807:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764055.101810:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764055.101812:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764055.101815:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832686592, offset 0
00000400:00000200:2.0:1787764055.101820:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764055.101824:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764055.101828:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764055.101832:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764055.101834:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764055.101837:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764055.101839:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764055.101842:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58599:58599:256:4294967295] 192.168.203.110@tcp LPNI seq info [58599:58599:8:4294967295]
00000400:00000200:2.0:1787764055.101863:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764055.101870:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764055.101875:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764055.101879:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764055.101883:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764055.101886:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787764055.101892:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764055.101895:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1787764055.101900:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764055.101902:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764055.101905:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764055.101909:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764055.101912:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764055.101914:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764055.101917:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764055.101921:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930b40 x1874599832686592/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764080 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.101930:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764055.101933:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930b40 x1874599832686592/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764080 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.101941:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764055.101943:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764055.101946:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764055.101950:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764055.101953:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764055.101956:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316442144
00000100:00000001:2.0:1787764055.101958:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316443380
00000100:00000001:2.0:1787764055.101961:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316442144
00000100:00000001:2.0:1787764055.101963:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316443380
00000100:00000001:2.0:1787764055.101965:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764055.101967:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764055.101970:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764055.101974:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764055/real 1787764055]  req@ffff9751d7930b40 x1874599832686592/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764055.101982:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764055.101984:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764055.101986:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764055.101988:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764055.101990:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764055.101993:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764055.101996:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930b40 x1874599832686592/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764055.102006:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930b40 x1874599832686592/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.102019:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764055.102021:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764055.102023:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764055.102024:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764055.102028:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930b40 x1874599832686592/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.102036:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764055.102041:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930b40 x1874599832686592/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.102048:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764055.102051:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764055.102053:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764055.102054:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764055.102056:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764055.102069:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764055.102071:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764055.102075:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764055.102077:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764055.102081:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930b40 x1874599832686592/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764055.102091:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832686592:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764055.102095:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764055.102099:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930000 x1874599832686720/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.102107:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764055.102108:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764055.102111:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764055.102113:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764055.102115:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764055.102118:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832686720:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764055.102122:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764055.102124:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764055.102126:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764055.102128:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764055.102132:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764055.102167:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764055.102172:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764055.102175:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764055.102179:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764055.102182:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832686720, portal 10
00000100:00000001:2.0:1787764055.102184:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764055.102185:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497721344 : -115097211830272 : ffff9751d7930000)
00000100:00000040:2.0:1787764055.102191:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930000 x1874599832686720/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764080 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.102198:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764055.102201:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764055.102203:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764055.102206:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832686720, offset 0
00000400:00000200:2.0:1787764055.102210:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764055.102214:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764055.102218:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764055.102222:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764055.102225:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764055.102227:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764055.102229:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764055.102233:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58600:58600:256:4294967295] 192.168.203.110@tcp LPNI seq info [58600:58600:8:4294967295]
00000400:00000200:2.0:1787764055.102237:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764055.102244:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764055.102250:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764055.102254:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764055.102258:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764055.102261:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787764055.102266:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764055.102269:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1787764055.102273:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764055.102275:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764055.102278:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764055.102282:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764055.102285:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764055.102287:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764055.102290:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764055.102294:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930000 x1874599832686720/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764080 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.102303:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764055.102306:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930000 x1874599832686720/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764080 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.102313:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764055.102315:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764055.102318:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764055.102322:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764055.102326:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764055.102328:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316442133
00000100:00000001:2.0:1787764055.102330:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316443380
00000100:00000001:2.0:1787764055.102333:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316442133
00000100:00000001:2.0:1787764055.102335:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316443380
00000100:00000001:2.0:1787764055.102337:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764055.102339:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764055.102341:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764055.102345:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764055/real 1787764055]  req@ffff9751d7930000 x1874599832686720/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764055.102354:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764055.102356:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764055.102358:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764055.102361:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764055.102362:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764055.102365:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764055.102368:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930000 x1874599832686720/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764055.102377:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930000 x1874599832686720/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.102385:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764055.102386:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764055.102388:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764055.102389:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764055.102393:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930000 x1874599832686720/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.102401:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764055.102405:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930000 x1874599832686720/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.102412:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764055.102415:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764055.102417:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764055.102418:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764055.102420:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764055.102422:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764055.102425:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764055.102428:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764055.102431:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764055.102436:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930000 x1874599832686720/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764055.102444:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832686720:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764055.102448:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764055.102452:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933480 x1874599832686848/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.102460:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764055.102461:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764055.102463:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764055.102465:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764055.102467:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764055.102470:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832686848:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764055.102474:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764055.102476:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764055.102478:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764055.102480:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764055.102484:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764055.102486:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764055.102489:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764055.102492:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764055.102496:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764055.102499:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832686848, portal 4
00000100:00000001:2.0:1787764055.102501:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764055.102502:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497734784 : -115097211816832 : ffff9751d7933480)
00000100:00000040:2.0:1787764055.102508:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933480 x1874599832686848/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764080 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.102515:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764055.102518:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764055.102520:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764055.102523:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832686848, offset 0
00000400:00000200:2.0:1787764055.102528:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764055.102532:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764055.102535:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764055.102538:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764055.102541:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764055.102544:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764055.102546:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764055.102549:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58601:58601:256:4294967295] 192.168.203.110@tcp LPNI seq info [58601:58601:8:4294967295]
00000400:00000200:2.0:1787764055.102554:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764055.102560:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764055.102565:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764055.102568:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764055.102572:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764055.102575:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787764055.102579:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764055.102582:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1787764055.102586:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764055.102588:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764055.102591:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764055.102595:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764055.102598:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764055.102600:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764055.102603:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764055.102606:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933480 x1874599832686848/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764080 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.102615:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764055.102618:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933480 x1874599832686848/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764080 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.102626:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764055.102627:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764055.102630:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764055.102633:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764055.102637:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764055.102639:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316442150
00000100:00000001:2.0:1787764055.102642:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316443380
00000100:00000001:2.0:1787764055.102645:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316442150
00000100:00000001:2.0:1787764055.102647:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316443380
00000100:00000001:2.0:1787764055.102649:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764055.102651:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764055.102653:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764055.102657:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764055/real 1787764055]  req@ffff9751d7933480 x1874599832686848/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764055.102665:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764055.102667:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764055.102669:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764055.102671:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764055.102673:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764055.102675:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764055.102678:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933480 x1874599832686848/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764055.102688:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933480 x1874599832686848/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.102694:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764055.102695:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764055.102697:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764055.102698:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764055.102702:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933480 x1874599832686848/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.102710:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764055.102714:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933480 x1874599832686848/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.102722:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764055.102725:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764055.102727:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764055.102728:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764055.102730:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764055.102732:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764055.102735:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764055.102738:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764055.102741:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764055.102745:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933480 x1874599832686848/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764055.102755:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832686848:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764055.102758:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764055.102760:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764055.102763:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930b40 x1874599832686592/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.102770:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764055.102772:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764055.102774:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764055.102777:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764055.102778:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764055.102780:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764055.102782:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764055.102784:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8c00.
02000000:00000001:2.0:1787764055.102787:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764055.102788:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764055.102790:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930b40.
00000100:00000001:2.0:1787764055.102793:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764055.102794:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764055.102796:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764055.102799:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930000 x1874599832686720/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.102813:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764055.102815:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764055.102816:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764055.102819:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764055.102820:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764055.102821:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764055.102823:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764055.102824:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8800.
02000000:00000001:2.0:1787764055.102826:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764055.102827:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764055.102829:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930000.
00000100:00000001:2.0:1787764055.102831:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764055.102832:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764055.102833:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764055.102837:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933480 x1874599832686848/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764080 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764055.102844:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764055.102860:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764055.102862:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764055.102865:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764055.102866:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764055.102868:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764055.102870:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764055.102871:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb800.
02000000:00000001:2.0:1787764055.102874:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764055.102875:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764055.102877:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933480.
00000100:00000001:2.0:1787764055.102880:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764055.102881:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764055.102884:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764055.102887:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764055.102889:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764055.102891:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764055.102892:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764055.740215:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764055.740219:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764055.740224:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764055.740224:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764055.740227:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764055.740232:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764055.740233:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764055.740235:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764055.740237:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764055.740243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764055.740244:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764055.740247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764055.740249:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764055.740254:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764055.740257:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764055.740310:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764055.740314:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764055.740318:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764056.124202:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764056.124212:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764056.124214:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764056.124219:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764056.124223:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764056.124227:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764056.124230:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764056.124233:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764056.124236:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764056.380219:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316443380, at 4316443392
00000100:00000010:3.0:1787764056.380236:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb70c0.
00000100:00000040:2.0:1787764056.380249:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316443380, at 4316443392
00000100:00000010:2.0:1787764056.380258:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7720.
00000100:00000001:1.0:1787764056.764241:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764056.764249:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764056.764255:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764056.764257:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764056.764260:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764056.764262:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764056.764265:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764056.764266:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764056.764268:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764056.764273:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764056.764278:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764056.764282:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764056.764285:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764056.764288:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764056.764291:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764056.764305:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764056.764308:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764056.764311:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787764057.148297:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787764057.148305:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764057.148306:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764057.148310:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787764057.148312:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764057.148315:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764057.148317:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764057.148319:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764057.148321:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764057.788236:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764057.788238:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764057.788244:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764057.788246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764057.788248:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764057.788255:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764057.788256:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764057.788261:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:1.0:1787764057.788262:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764057.788266:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:1.0:1787764057.788267:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764057.788271:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764057.788272:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764057.788278:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764057.788278:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764057.788283:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764057.788286:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764057.788289:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764057.788292:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764057.788295:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764057.788298:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764057.788350:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764057.788354:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764057.788357:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787764058.172246:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787764058.172255:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764058.172257:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764058.172264:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787764058.172267:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764058.172270:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764058.172273:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764058.172276:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764058.172278:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764058.812243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764058.812248:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764058.812255:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764058.812261:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764058.812264:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764058.812265:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764058.812269:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764058.812269:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764058.812277:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764058.812280:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764058.812282:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764058.812283:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764058.812289:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764058.812292:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764058.812295:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764058.812340:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764058.812347:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764058.812352:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787764059.196283:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787764059.196292:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764059.196294:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764059.196299:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787764059.196302:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764059.196306:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764059.196309:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764059.196311:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764059.196313:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764059.836227:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764059.836236:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764059.836238:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764059.836244:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764059.836252:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764059.836258:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764059.836258:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764059.836263:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764059.836265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764059.836269:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764059.836274:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764059.836275:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764059.836279:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764059.836288:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764059.836291:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764059.836334:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764059.836339:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764059.836343:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764060.220290:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764060.220300:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764060.220302:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764060.220307:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764060.220311:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764060.220314:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764060.220317:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764060.220320:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764060.220323:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764060.220326:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764060.220332:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764060.220334:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764060.220338:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764060.220344:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764060.220352:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21774
00000400:00000200:3.0:1787764060.220360:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764060.220366:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764060.220369:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764060.220372:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764060.220375:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764060.220377:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764060.220379:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764060.220383:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764060.220384:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787764060.220387:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764060.220390:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787764060.220392:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787764060.220393:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787764060.220395:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764060.220397:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764060.220398:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764060.220400:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764060.220402:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787764060.220405:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764060.220407:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764060.220409:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764060.220412:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764060.220415:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764060.220417:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764060.220419:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764060.220429:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bcf00.
00000020:00000040:3.0:1787764060.220432:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787764060.220436:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764060.220438:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764060.220441:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764060.220444:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764060.220448:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764060.220455:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612000.
00000100:00000001:3.0:1787764060.220462:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764060.220471:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bcf00 x1874599832686976/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764060.220480:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764060.220484:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764060.220488:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ebc1bc5] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bcf00 x1874599832686976/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764060.220508:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764060.220510:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764060.220513:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787764060.220514:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764060.220516:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764060.220518:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764060.220521:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21774
00000400:00000200:3.0:1787764060.220524:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764060.220526:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764060.220528:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764060.220531:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764060.220532:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764060.220534:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764060.220536:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764060.220537:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1787764060.220538:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:3.0:1787764060.220539:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:3.0:1787764060.220540:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764060.220549:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000001:2.0:1787764060.220552:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1787764060.220552:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:3.0:1787764060.220560:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787764060.220561:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764060.220569:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764060.220572:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764060.220578:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000040:2.0:1787764060.220580:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bcf00 x1874599832686976/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764060.220587:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764060.220600:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00000001:2.0:1787764060.220602:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764060.220605:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764060.220612:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:3.0:1787764060.220612:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:2.0:1787764060.220617:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764060.220620:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764060.220621:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764060.220628:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:2.0:1787764060.220630:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832686976:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764060.220636:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764060.220638:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764060.220640:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764060.220642:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764060.220644:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764060.220645:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1787764060.220650:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:3.0:1787764060.220654:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000010:2.0:1787764060.220659:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764060.220662:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:3.0:1787764060.220665:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764060.220668:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764060.220669:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764060.220678:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000100:00000010:3.0:1787764060.220678:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc780.
00000020:00000040:3.0:1787764060.220683:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787764060.220686:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764060.220687:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764060.220688:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000001:3.0:1787764060.220691:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764060.220693:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764060.220697:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764060.220698:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832686976, portal 25
00000100:00000001:2.0:1787764060.220707:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764060.220710:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049849600 : -115096659702016 : ffff9751f87bcf00)
02000000:00000010:3.0:1787764060.220710:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610800.
00000100:00000001:3.0:1787764060.220724:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764060.220725:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bcf00 x1874599832686976/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764085 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764060.220740:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc780 x1874599832687104/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764060.220741:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764060.220756:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764060.220765:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000001:3.0:1787764060.220766:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:2.0:1787764060.220776:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832686976, offset 0
00000100:00000001:3.0:1787764060.220776:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764060.220793:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7d56184] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc780 x1874599832687104/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764060.220794:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787764060.220808:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764060.220809:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764060.220828:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00080000:3.0:1787764060.220831:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787764060.220835:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:3.0:1787764060.220837:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1787764060.220841:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:2.0:1787764060.220846:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00080000:3.0:1787764060.220847:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1787764060.220853:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1787764060.220855:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:2.0:1787764060.220858:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00080000:3.0:1787764060.220862:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21774
00000400:00000200:2.0:1787764060.220867:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58602:58602:256:4294967295] 192.168.203.110@tcp LPNI seq info [58602:58602:8:4294967295]
00000400:00000200:2.0:1787764060.220871:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764060.220878:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000400:00000200:3.0:1787764060.220887:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:2.0:1787764060.220888:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764060.220898:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000800:00000010:2.0:1787764060.220899:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000100:00080000:3.0:1787764060.220905:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764060.220909:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:2.0:1787764060.220913:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787764060.220916:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1787764060.220919:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000100:00000040:3.0:1787764060.220924:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764060.220931:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764060.220939:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:2.1:1787764060.220939:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:3.0:1787764060.220948:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000800:00000200:2.1:1787764060.220949:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000020:00000001:3.0:1787764060.220951:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764060.220956:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000100:2.0:1787764060.220959:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000020:00000001:3.0:1787764060.220964:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000010:2.0:1787764060.220965:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000020:00000001:3.0:1787764060.220975:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:2.0:1787764060.220978:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1787764060.220985:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764060.220992:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1787764060.220994:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:3.0:1787764060.221004:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764060.221005:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000100:00000001:3.0:1787764060.221009:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000010:2.0:1787764060.221011:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000020:00000040:3.0:1787764060.221014:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00000001:2.0:1787764060.221018:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:3.0:1787764060.221018:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764060.221023:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764060.221027:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bcf00 x1874599832686976/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764085 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787764060.221028:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764060.221035:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764060.221039:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764060.221040:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787764060.221042:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787764060.221049:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764060.221059:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764060.221068:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc3c0.
00000020:00000040:3.0:1787764060.221077:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000001:2.0:1787764060.221080:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:3.0:1787764060.221081:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764060.221087:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764060.221091:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bcf00 x1874599832686976/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764085 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764060.221092:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764060.221098:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764060.221104:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764060.221114:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764060.221116:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610c00.
00000100:00000001:2.0:1787764060.221118:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764060.221128:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764060.221130:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764060.221170:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764060.221172:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316448500
00000100:00100000:3.0:1787764060.221179:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc3c0 x1874599832687232/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:2.0:1787764060.221182:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7720.
00000100:00000001:2.0:1787764060.221192:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787764060.221192:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764060.221196:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764060.221204:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316448500
00000100:00000040:3.0:1787764060.221205:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000060bf8470] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc3c0 x1874599832687232/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1787764060.221218:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb70c0.
00000100:00000001:3.0:1787764060.221220:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764060.221223:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000040:3.0:1787764060.221224:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764060.221228:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1787764060.221228:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1787764060.221236:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764060.221239:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764060.221240:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00080000:3.0:1787764060.221247:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787764060.221253:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:2.0:1787764060.221254:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764060/real 1787764060]  req@ffff9751f87bcf00 x1874599832686976/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787764060.221255:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764060.221261:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764060.221271:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21774
00000400:00000200:3.0:1787764060.221290:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000001:2.0:1787764060.221295:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000200:3.0:1787764060.221302:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764060.221305:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000001:2.0:1787764060.221308:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:3.0:1787764060.221309:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000010:2.0:1787764060.221312:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000100:00000001:3.0:1787764060.221313:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1787764060.221322:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000100:00000040:3.0:1787764060.221323:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000010:2.0:1787764060.221329:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:3.0:1787764060.221330:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764060.221340:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1787764060.221341:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000040:3.0:1787764060.221343:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:3.0:1787764060.221347:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000200:2.0:1787764060.221348:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bcf00 x1874599832686976/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787764060.221350:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:3.0:1787764060.221358:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:3.0:1787764060.221369:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:3.0:1787764060.221381:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000200:2.0:1787764060.221387:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bcf00 x1874599832686976/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764060.221387:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764060.221396:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764060.221402:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764060.221412:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:2.0:1787764060.221413:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764060.221419:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1787764060.221421:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764060.221430:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764060.221430:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764060.221436:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787764060.221437:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764060.221446:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764060.221450:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764060.221451:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bcf00 x1874599832686976/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764060.221452:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787764060.221458:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1787764060.221467:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764060.221472:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764060.221480:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bfc00.
00000020:00000040:3.0:1787764060.221486:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:3.0:1787764060.221492:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1787764060.221493:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787764060.221494:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764060.221501:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764060.221503:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764060.221504:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764060.221507:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611c00.
00000100:00000040:2.0:1787764060.221510:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bcf00 x1874599832686976/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764060.221512:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764060.221521:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bfc00 x1874599832687360/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764060.221554:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764060.221556:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764060.221560:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764060.221563:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764060.221569:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:3.0:1787764060.221570:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003c1f6b31] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bfc00 x1874599832687360/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.221574:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764060.221582:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764060.221587:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764060.221597:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:3.0:1787764060.221607:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764060.221608:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764060.221617:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764060.221624:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bcf00 x1874599832686976/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787764060.221624:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21784)
00000100:00100000:2.0:1787764060.221638:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832686976:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764060.221644:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764060.221647:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764060.221651:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bcf00 x1874599832686976/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.221657:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764060.221660:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764060.221664:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764060.221667:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764060.221670:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764060.221673:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764060.221675:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764060.221677:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612000.
02000000:00000001:2.0:1787764060.221681:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764060.221683:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764060.221685:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bcf00.
00000100:00000001:2.0:1787764060.221689:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764060.221690:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764060.221693:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764060.221700:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764060.221702:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764060.221705:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764060.221707:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764060.221708:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764060.221712:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc780 x1874599832687104/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.221718:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764060.221719:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764060.221721:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764060.221723:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764060.221724:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764060.221726:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832687104:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764060.221728:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764060.221730:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764060.221732:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764060.221733:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764060.221735:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764060.221736:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764060.221740:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764060.221742:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764060.221745:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764060.221747:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832687104, portal 10
00000100:00000001:2.0:1787764060.221749:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764060.221750:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049847680 : -115096659703936 : ffff9751f87bc780)
00000100:00000040:2.0:1787764060.221754:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc780 x1874599832687104/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764085 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.221759:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764060.221760:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764060.221762:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764060.221763:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832687104, offset 0
00000400:00000200:2.0:1787764060.221766:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764060.221769:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764060.221772:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764060.221774:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764060.221776:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764060.221778:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764060.221779:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764060.221782:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58603:58603:256:4294967295] 192.168.203.110@tcp LPNI seq info [58603:58603:8:4294967295]
00000400:00000200:2.0:1787764060.221785:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764060.221789:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764060.221792:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764060.221795:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764060.221798:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764060.221800:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787764060.221803:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764060.221806:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:2.0:1787764060.221809:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764060.221825:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764060.221829:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764060.221831:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764060.221833:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764060.221835:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764060.221837:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764060.221839:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc780 x1874599832687104/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764085 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.221846:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764060.221850:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc780 x1874599832687104/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764085 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.221863:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764060.221865:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764060.221868:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764060.221872:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764060.221876:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764060.221878:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316447264
00000100:00000001:2.0:1787764060.221881:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316448500
00000100:00000001:2.0:1787764060.221884:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316447264
00000100:00000001:2.0:1787764060.221886:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316448500
00000100:00000001:2.0:1787764060.221888:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764060.221890:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764060.221892:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764060.221897:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764060/real 1787764060]  req@ffff9751f87bc780 x1874599832687104/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764060.221906:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764060.221908:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764060.221910:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764060.221912:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764060.221914:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764060.221917:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764060.221920:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc780 x1874599832687104/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764060.221930:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc780 x1874599832687104/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.221937:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764060.221938:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764060.221940:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764060.221941:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764060.221945:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc780 x1874599832687104/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.221952:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764060.221957:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc780 x1874599832687104/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.221964:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764060.221966:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764060.221968:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764060.221969:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764060.221971:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764060.221985:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764060.221988:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764060.221992:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764060.221995:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764060.221999:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc780 x1874599832687104/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764060.222007:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832687104:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764060.222012:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764060.222016:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc3c0 x1874599832687232/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.222023:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764060.222025:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764060.222027:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764060.222029:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764060.222032:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764060.222035:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832687232:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764060.222039:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764060.222041:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764060.222043:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764060.222045:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764060.222049:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764060.222051:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764060.222054:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764060.222057:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764060.222061:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764060.222064:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832687232, portal 10
00000100:00000001:2.0:1787764060.222066:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764060.222067:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049846720 : -115096659704896 : ffff9751f87bc3c0)
00000100:00000040:2.0:1787764060.222073:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc3c0 x1874599832687232/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764085 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.222081:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764060.222084:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764060.222086:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764060.222088:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832687232, offset 0
00000400:00000200:2.0:1787764060.222093:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764060.222097:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764060.222101:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764060.222113:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764060.222116:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764060.222122:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764060.222125:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764060.222128:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58604:58604:256:4294967295] 192.168.203.110@tcp LPNI seq info [58604:58604:8:4294967295]
00000400:00000200:2.0:1787764060.222132:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764060.222160:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764060.222166:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764060.222170:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764060.222174:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764060.222177:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787764060.222181:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764060.222184:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:2.0:1787764060.222189:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764060.222191:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764060.222194:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764060.222198:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764060.222202:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764060.222204:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764060.222207:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764060.222210:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc3c0 x1874599832687232/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764085 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.222219:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764060.222222:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc3c0 x1874599832687232/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764085 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.222229:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764060.222231:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764060.222235:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764060.222238:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764060.222242:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764060.222244:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316447253
00000100:00000001:2.0:1787764060.222246:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316448500
00000100:00000001:2.0:1787764060.222249:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316447253
00000100:00000001:2.0:1787764060.222251:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316448500
00000100:00000001:2.0:1787764060.222253:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764060.222255:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764060.222258:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764060.222262:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764060/real 1787764060]  req@ffff9751f87bc3c0 x1874599832687232/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764060.222271:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764060.222272:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764060.222274:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764060.222277:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764060.222279:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764060.222282:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764060.222285:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc3c0 x1874599832687232/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764060.222294:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc3c0 x1874599832687232/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.222302:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764060.222304:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764060.222306:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764060.222307:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764060.222311:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc3c0 x1874599832687232/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.222318:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764060.222322:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc3c0 x1874599832687232/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.222330:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764060.222333:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764060.222334:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764060.222336:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764060.222338:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764060.222339:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764060.222342:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764060.222345:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764060.222348:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764060.222352:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc3c0 x1874599832687232/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764060.222360:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832687232:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764060.222364:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764060.222369:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bfc00 x1874599832687360/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.222376:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764060.222378:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764060.222380:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764060.222382:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764060.222384:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764060.222387:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832687360:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764060.222391:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764060.222393:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764060.222395:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764060.222397:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764060.222400:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764060.222403:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764060.222406:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764060.222409:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764060.222413:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764060.222416:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832687360, portal 4
00000100:00000001:2.0:1787764060.222418:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764060.222420:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049861120 : -115096659690496 : ffff9751f87bfc00)
00000100:00000040:2.0:1787764060.222426:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bfc00 x1874599832687360/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764085 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.222434:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764060.222436:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764060.222439:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764060.222441:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832687360, offset 0
00000400:00000200:2.0:1787764060.222446:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764060.222450:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764060.222454:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764060.222457:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764060.222460:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764060.222462:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764060.222465:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764060.222469:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58605:58605:256:4294967295] 192.168.203.110@tcp LPNI seq info [58605:58605:8:4294967295]
00000400:00000200:2.0:1787764060.222474:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764060.222481:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764060.222486:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764060.222489:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764060.222493:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764060.222496:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000200:2.1:1787764060.222500:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764060.222504:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 28 secs later)
00000800:00000100:2.0:1787764060.222508:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764060.222511:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764060.222514:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764060.222517:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764060.222520:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764060.222522:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764060.222525:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764060.222529:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bfc00 x1874599832687360/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764085 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.222537:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764060.222540:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bfc00 x1874599832687360/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764085 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.222547:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764060.222548:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764060.222551:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764060.222554:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764060.222558:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764060.222560:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316447270
00000100:00000001:2.0:1787764060.222563:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316448500
00000100:00000001:2.0:1787764060.222565:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316447270
00000100:00000001:2.0:1787764060.222568:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316448500
00000100:00000001:2.0:1787764060.222570:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764060.222573:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764060.222575:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764060.222579:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764060/real 1787764060]  req@ffff9751f87bfc00 x1874599832687360/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764060.222588:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764060.222590:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764060.222591:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764060.222594:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764060.222595:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764060.222597:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764060.222601:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bfc00 x1874599832687360/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764060.222609:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bfc00 x1874599832687360/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.222616:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764060.222617:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764060.222619:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764060.222621:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764060.222624:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bfc00 x1874599832687360/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.222631:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764060.222635:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bfc00 x1874599832687360/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.222642:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764060.222644:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764060.222646:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764060.222648:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764060.222650:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764060.222651:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764060.222654:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764060.222658:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764060.222660:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764060.222665:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bfc00 x1874599832687360/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764060.222672:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832687360:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764060.222676:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764060.222678:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764060.222681:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc780 x1874599832687104/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.222688:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764060.222690:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764060.222692:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764060.222695:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764060.222697:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764060.222699:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764060.222701:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764060.222703:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610800.
02000000:00000001:2.0:1787764060.222705:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764060.222707:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764060.222708:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc780.
00000100:00000001:2.0:1787764060.222710:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764060.222712:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764060.222713:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764060.222716:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc3c0 x1874599832687232/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.222723:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764060.222724:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764060.222726:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764060.222728:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764060.222729:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764060.222731:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764060.222733:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764060.222735:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610c00.
02000000:00000001:2.0:1787764060.222737:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764060.222739:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764060.222740:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc3c0.
00000100:00000001:2.0:1787764060.222742:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764060.222744:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764060.222745:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764060.222749:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bfc00 x1874599832687360/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764085 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764060.222757:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764060.222758:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764060.222760:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764060.222762:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764060.222763:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764060.222764:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764060.222766:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764060.222768:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611c00.
02000000:00000001:2.0:1787764060.222770:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764060.222771:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764060.222772:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bfc00.
00000100:00000001:2.0:1787764060.222775:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764060.222776:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764060.222778:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764060.222782:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764060.222784:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764060.222786:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764060.222788:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764060.860278:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764060.860287:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764060.860291:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764060.860297:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764060.860298:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764060.860303:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764060.860305:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764060.860308:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764060.860311:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764060.860315:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764060.860315:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764060.860320:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764060.860322:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764060.860329:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764060.860334:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764060.860369:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764060.860374:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764060.860378:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764061.244234:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764061.244244:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764061.244246:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764061.244252:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764061.244256:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764061.244259:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764061.244261:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764061.244264:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764061.244266:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764061.500217:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316448500, at 4316448512
00000100:00000010:3.0:1787764061.500233:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb70c0.
00000100:00000040:2.0:1787764061.500270:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316448500, at 4316448512
00000100:00000010:2.0:1787764061.500279:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7720.
00000100:00000001:1.0:1787764061.884219:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764061.884219:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764061.884228:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764061.884231:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764061.884235:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764061.884237:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764061.884241:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764061.884243:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764061.884246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764061.884249:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764061.884251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764061.884255:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764061.884256:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764061.884260:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:1.0:1787764061.884261:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764061.884263:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764061.884267:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:1.0:1787764061.884269:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764061.884270:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764061.884275:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764061.884278:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764061.884316:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764061.884320:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764061.884323:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764062.268271:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764062.268281:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764062.268284:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764062.268290:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764062.268295:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764062.268298:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764062.268301:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764062.268304:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764062.268307:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764062.908249:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764062.908250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764062.908257:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764062.908259:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764062.908262:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764062.908264:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764062.908265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764062.908272:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764062.908272:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764062.908274:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764062.908279:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764062.908284:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764062.908287:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764062.908290:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764062.908293:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764062.908317:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764062.908322:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764062.908325:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764063.292276:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764063.292285:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764063.292287:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764063.292293:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764063.292297:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764063.292300:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764063.292303:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764063.292306:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764063.292309:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764063.932227:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764063.932228:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764063.932233:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764063.932236:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764063.932239:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764063.932241:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764063.932242:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764063.932250:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764063.932250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764063.932251:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764063.932258:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764063.932262:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764063.932266:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764063.932269:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764063.932273:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764063.932288:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764063.932294:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764063.932298:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764064.316229:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764064.316236:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764064.316238:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764064.316242:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764064.316246:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764064.316249:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764064.316251:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764064.316255:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764064.316258:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764064.956237:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764064.956244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764064.956245:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764064.956248:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764064.956252:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00100000:1.0:1787764064.956254:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764064.956257:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:1.0:1787764064.956259:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764064.956262:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764064.956263:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764064.956270:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764064.956270:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:1.0:1787764064.956273:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764064.956276:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764064.956282:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764064.956286:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764064.956289:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764064.956294:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764064.956296:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764064.956300:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764064.956302:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764064.956341:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764064.956345:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764064.956348:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764065.340285:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764065.340296:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764065.340299:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764065.340306:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:0.0:1787764065.340307:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787764065.340312:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1787764065.340316:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787764065.340320:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1787764065.340321:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787764065.340326:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764065.340326:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764065.340329:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764065.340336:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764065.340341:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764065.340351:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21779
00000400:00000200:0.0:1787764065.340361:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764065.340369:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764065.340373:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764065.340377:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764065.340380:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764065.340382:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764065.340385:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764065.340390:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764065.340392:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764065.340394:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764065.340398:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764065.340401:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764065.340404:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787764065.340407:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764065.340410:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764065.340412:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764065.340418:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764065.340421:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787764065.340426:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764065.340429:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764065.340432:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764065.340436:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764065.340439:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764065.340442:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764065.340445:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764065.340455:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000020:00000040:0.0:1787764065.340459:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764065.340463:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764065.340465:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764065.340468:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764065.340471:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764065.340475:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764065.340483:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787764065.340492:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764065.340502:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832687488/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764065.340514:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764065.340519:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764065.340525:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832687488/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764065.340560:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764065.340563:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764065.340567:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764065.340568:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764065.340570:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764065.340573:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764065.340576:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21779
00000400:00000200:0.0:1787764065.340588:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764065.340591:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764065.340593:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764065.340596:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764065.340598:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764065.340600:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764065.340602:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764065.340604:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764065.340606:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787764065.340607:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764065.340609:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787764065.340611:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787764065.340613:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787764065.340616:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764065.340618:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764065.340620:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764065.340622:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764065.340625:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787764065.340627:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764065.340630:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764065.340631:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764065.340636:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764065.340638:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764065.340640:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764065.340644:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787764065.340648:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764065.340650:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764065.340654:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
00000020:00000040:0.0:1787764065.340657:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787764065.340659:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764065.340661:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764065.340663:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764065.340664:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764065.340666:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764065.340668:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000010:0.0:1787764065.340673:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787764065.340678:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764065.340683:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832687616/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764065.340684:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764065.340689:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1787764065.340695:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764065.340699:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764065.340702:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cb40 x1874599832687488/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764065.340707:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832687616/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.340728:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764065.340732:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764065.340733:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764065.340737:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:0.0:1787764065.340738:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764065.340743:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
02000000:00000001:2.0:1787764065.340743:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764065.340747:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
02000000:00000001:2.0:1787764065.340749:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764065.340751:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764065.340755:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00100000:2.0:1787764065.340758:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832687488:192.168.203.110@tcp:250:kworker.0
00000100:00080000:0.0:1787764065.340760:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21779
00000100:00000001:2.0:1787764065.340766:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000400:00000200:0.0:1787764065.340768:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764065.340773:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
02000000:00000001:2.0:1787764065.340773:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:0.0:1787764065.340790:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
02000000:00000001:2.0:1787764065.340792:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764065.340796:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764065.340798:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1787764065.340802:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764065.340806:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000010:2.0:1787764065.340808:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
00000100:00000001:0.0:1787764065.340809:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:2.0:1787764065.340813:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764065.340814:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764065.340818:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787764065.340820:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764065.340822:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000400:00000200:2.0:1787764065.340825:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:0.0:1787764065.340826:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787764065.340830:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000010:2.0:1787764065.340832:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000100:00000040:0.0:1787764065.340834:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764065.340838:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764065.340840:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000010:2.0:1787764065.340842:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000001:0.0:1787764065.340843:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764065.340848:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787764065.340850:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764065.340854:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764065.340854:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832687488, portal 25
02000000:00000001:0.0:1787764065.340858:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787764065.340861:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:0.0:1787764065.340863:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764065.340865:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049520960 : -115096660030656 : ffff9751f876cb40)
02000000:00000001:0.0:1787764065.340867:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764065.340871:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764065.340874:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787764065.340878:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764065.340878:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cb40 x1874599832687488/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764090 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764065.340883:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764065.340891:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000100:00000001:2.0:1787764065.340893:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000040:0.0:1787764065.340895:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000040:2.0:1787764065.340900:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
02000000:00000001:0.0:1787764065.340902:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764065.340905:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764065.340906:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000001:0.0:1787764065.340909:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:2.0:1787764065.340911:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832687488, offset 0
02000000:00000001:0.0:1787764065.340912:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764065.340916:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764065.340922:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000400:00000200:2.0:1787764065.340923:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787764065.340928:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764065.340937:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00100000:0.0:1787764065.340938:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832687744/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787764065.340948:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:0.0:1787764065.340949:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764065.340953:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764065.340958:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:0.0:1787764065.340961:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832687744/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764065.340967:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:0.0:1787764065.340975:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764065.340976:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000040:0.0:1787764065.340980:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787764065.340981:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000040:0.0:1787764065.340988:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1787764065.340989:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58606:58606:256:4294967295] 192.168.203.110@tcp LPNI seq info [58606:58606:8:4294967295]
00000100:00080000:0.0:1787764065.340994:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787764065.340999:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:0.0:1787764065.341002:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787764065.341005:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764065.341007:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764065.341012:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764065.341017:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21779
00000400:00000200:2.0:1787764065.341020:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000400:00000200:0.0:1787764065.341026:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:2.0:1787764065.341032:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787764065.341033:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764065.341037:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000010:2.0:1787764065.341039:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000100:00000040:0.0:1787764065.341040:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764065.341045:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764065.341049:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764065.341055:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:2.0:1787764065.341055:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787764065.341059:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:2.0:1787764065.341063:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000020:00000040:0.0:1787764065.341064:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787764065.341068:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764065.341071:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000800:00000200:2.1:1787764065.341074:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787764065.341076:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787764065.341081:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000800:00000200:2.1:1787764065.341082:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000100:00000040:0.0:1787764065.341089:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764065.341094:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000100:2.0:1787764065.341095:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787764065.341099:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000010:2.0:1787764065.341101:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000100:00000001:0.0:1787764065.341103:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764065.341108:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000400:00000200:2.0:1787764065.341112:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:0.0:1787764065.341114:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764065.341117:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764065.341119:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764065.341204:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764065.341208:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764065.341210:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764065.341215:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787764065.341217:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764065.341218:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764065.341224:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787764065.341227:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787764065.341230:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764065.341231:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764065.341233:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764065.341234:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764065.341236:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764065.341240:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787764065.341244:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764065.341252:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832687872/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764065.341261:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764065.341263:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764065.341269:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832687872/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764065.341277:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764065.341281:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21790)
00000400:00000200:2.0:1787764065.341324:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764065.341330:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764065.341332:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764065.341338:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764065.341343:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cb40 x1874599832687488/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764090 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.341353:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764065.341357:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cb40 x1874599832687488/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764090 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.341364:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764065.341366:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764065.341374:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764065.341394:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764065.341397:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316453620
00000100:00000010:2.0:1787764065.341405:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7720.
00000100:00000001:2.0:1787764065.341413:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787764065.341423:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316453620
00000100:00000010:2.0:1787764065.341429:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7240.
00000100:00000001:2.0:1787764065.341434:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787764065.341438:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764065.341442:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764065.341446:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764065.341454:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764065/real 1787764065]  req@ffff9751f876cb40 x1874599832687488/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764065.341466:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764065.341468:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764065.341470:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764065.341474:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764065.341476:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764065.341479:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764065.341483:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cb40 x1874599832687488/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764065.341491:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cb40 x1874599832687488/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.341497:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764065.341499:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764065.341502:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764065.341504:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764065.341508:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cb40 x1874599832687488/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.341514:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764065.341519:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cb40 x1874599832687488/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.341527:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764065.341530:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764065.341532:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764065.341533:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764065.341535:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764065.341537:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764065.341539:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764065.341543:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764065.341545:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764065.341560:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cb40 x1874599832687488/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764065.341569:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832687488:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764065.341573:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764065.341575:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764065.341578:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cb40 x1874599832687488/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.341584:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764065.341587:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764065.341589:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764065.341593:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764065.341595:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764065.341597:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764065.341599:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764065.341601:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016800.
02000000:00000001:2.0:1787764065.341605:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764065.341607:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764065.341609:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cb40.
00000100:00000001:2.0:1787764065.341612:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764065.341613:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764065.341615:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764065.341623:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764065.341625:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764065.341627:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764065.341629:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764065.341630:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764065.341634:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876da40 x1874599832687616/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.341641:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764065.341643:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764065.341645:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764065.341647:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764065.341649:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764065.341651:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832687616:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764065.341655:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764065.341657:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764065.341659:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764065.341660:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764065.341665:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764065.341668:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764065.341672:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764065.341675:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764065.341680:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764065.341683:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832687616, portal 10
00000100:00000001:2.0:1787764065.341685:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764065.341687:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049524800 : -115096660026816 : ffff9751f876da40)
00000100:00000040:2.0:1787764065.341696:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876da40 x1874599832687616/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764090 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.341702:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764065.341705:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764065.341707:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764065.341710:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832687616, offset 0
00000400:00000200:2.0:1787764065.341715:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764065.341720:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764065.341724:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764065.341727:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764065.341730:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764065.341733:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764065.341736:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764065.341740:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58607:58607:256:4294967295] 192.168.203.110@tcp LPNI seq info [58607:58607:8:4294967295]
00000400:00000200:2.0:1787764065.341745:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764065.341753:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764065.341758:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764065.341762:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764065.341767:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764065.341770:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787764065.341790:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764065.341794:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1787764065.341798:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764065.341801:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764065.341804:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764065.341807:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764065.341811:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764065.341812:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764065.341815:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764065.341819:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876da40 x1874599832687616/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764090 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.341827:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764065.341830:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876da40 x1874599832687616/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764090 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.341837:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764065.341838:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764065.341841:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764065.341845:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764065.341849:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764065.341851:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316452384
00000100:00000001:2.0:1787764065.341855:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316453620
00000100:00000001:2.0:1787764065.341858:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316452384
00000100:00000001:2.0:1787764065.341860:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316453620
00000100:00000001:2.0:1787764065.341862:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764065.341864:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764065.341867:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764065.341870:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764065/real 1787764065]  req@ffff9751f876da40 x1874599832687616/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764065.341878:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764065.341880:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764065.341882:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764065.341885:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764065.341886:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764065.341888:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764065.341891:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876da40 x1874599832687616/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764065.341899:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876da40 x1874599832687616/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.341911:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764065.341913:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764065.341915:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764065.341916:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764065.341919:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876da40 x1874599832687616/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.341926:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764065.341930:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876da40 x1874599832687616/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.341937:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764065.341939:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764065.341940:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764065.341942:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764065.341944:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764065.341959:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764065.341961:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764065.341964:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764065.341966:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764065.341969:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876da40 x1874599832687616/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764065.341977:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832687616:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764065.341980:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764065.341984:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876de00 x1874599832687744/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.341989:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764065.341991:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764065.341993:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764065.341994:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764065.341996:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764065.341998:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832687744:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764065.342001:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764065.342003:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764065.342005:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764065.342006:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764065.342010:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764065.342013:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764065.342016:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764065.342019:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764065.342021:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764065.342024:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832687744, portal 10
00000100:00000001:2.0:1787764065.342026:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764065.342028:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049525760 : -115096660025856 : ffff9751f876de00)
00000100:00000040:2.0:1787764065.342033:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876de00 x1874599832687744/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764090 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.342039:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764065.342041:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764065.342044:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764065.342047:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832687744, offset 0
00000400:00000200:2.0:1787764065.342051:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764065.342055:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764065.342059:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764065.342062:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764065.342064:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764065.342067:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764065.342069:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764065.342072:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58608:58608:256:4294967295] 192.168.203.110@tcp LPNI seq info [58608:58608:8:4294967295]
00000400:00000200:2.0:1787764065.342076:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764065.342082:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764065.342087:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764065.342091:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764065.342095:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764065.342097:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787764065.342101:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764065.342104:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1787764065.342108:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764065.342110:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764065.342124:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764065.342128:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764065.342132:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764065.342173:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764065.342179:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764065.342185:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876de00 x1874599832687744/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764090 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.342195:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764065.342199:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876de00 x1874599832687744/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764090 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.342206:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764065.342209:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764065.342212:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764065.342215:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764065.342219:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764065.342221:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316452373
00000100:00000001:2.0:1787764065.342224:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316453620
00000100:00000001:2.0:1787764065.342226:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316452373
00000100:00000001:2.0:1787764065.342229:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316453620
00000100:00000001:2.0:1787764065.342231:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764065.342233:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764065.342235:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764065.342239:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764065/real 1787764065]  req@ffff9751f876de00 x1874599832687744/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764065.342248:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764065.342249:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764065.342251:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764065.342253:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764065.342254:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764065.342256:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764065.342258:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876de00 x1874599832687744/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764065.342263:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876de00 x1874599832687744/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.342268:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764065.342269:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764065.342270:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764065.342271:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764065.342273:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876de00 x1874599832687744/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.342278:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764065.342281:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876de00 x1874599832687744/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.342285:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764065.342286:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764065.342287:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764065.342288:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764065.342289:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764065.342290:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764065.342292:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764065.342294:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764065.342295:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764065.342297:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876de00 x1874599832687744/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764065.342302:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832687744:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764065.342305:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764065.342307:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832687872/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.342312:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764065.342313:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764065.342314:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764065.342315:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764065.342317:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764065.342318:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832687872:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764065.342321:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764065.342322:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764065.342324:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764065.342325:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764065.342328:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764065.342329:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764065.342331:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764065.342333:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764065.342336:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764065.342338:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832687872, portal 4
00000100:00000001:2.0:1787764065.342339:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764065.342340:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
00000100:00000040:2.0:1787764065.342344:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832687872/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764090 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.342349:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764065.342350:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764065.342352:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764065.342353:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832687872, offset 0
00000400:00000200:2.0:1787764065.342356:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764065.342358:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764065.342361:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764065.342363:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764065.342365:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764065.342366:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764065.342368:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764065.342370:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58609:58609:256:4294967295] 192.168.203.110@tcp LPNI seq info [58609:58609:8:4294967295]
00000400:00000200:2.0:1787764065.342373:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764065.342377:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764065.342380:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764065.342382:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764065.342384:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764065.342386:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:2.1:1787764065.342389:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764065.342391:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:2.0:1787764065.342394:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764065.342396:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764065.342398:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764065.342401:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764065.342403:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764065.342404:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764065.342405:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764065.342408:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876c000 x1874599832687872/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764090 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.342413:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764065.342415:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832687872/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764090 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.342420:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764065.342421:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764065.342423:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764065.342425:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764065.342427:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764065.342428:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316452390
00000100:00000001:2.0:1787764065.342430:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316453620
00000100:00000001:2.0:1787764065.342431:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316452390
00000100:00000001:2.0:1787764065.342433:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316453620
00000100:00000001:2.0:1787764065.342434:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764065.342436:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764065.342437:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764065.342439:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764065/real 1787764065]  req@ffff9751f876c000 x1874599832687872/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764065.342443:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764065.342444:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764065.342445:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764065.342446:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764065.342447:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764065.342449:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764065.342451:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832687872/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764065.342456:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832687872/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.342460:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764065.342461:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764065.342462:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764065.342463:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764065.342465:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832687872/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.342469:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764065.342471:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832687872/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.342475:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764065.342476:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764065.342477:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764065.342478:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764065.342479:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764065.342480:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764065.342481:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764065.342483:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764065.342484:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764065.342487:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832687872/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764065.342492:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832687872:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764065.342494:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764065.342495:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764065.342498:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876da40 x1874599832687616/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.342502:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764065.342504:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764065.342505:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764065.342506:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764065.342507:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764065.342508:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764065.342510:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764065.342511:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015c00.
02000000:00000001:2.0:1787764065.342513:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764065.342514:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764065.342515:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876da40.
00000100:00000001:2.0:1787764065.342517:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764065.342518:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764065.342520:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764065.342521:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876de00 x1874599832687744/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.342526:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764065.342526:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764065.342527:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764065.342528:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764065.342529:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764065.342530:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764065.342531:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764065.342532:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:2.0:1787764065.342533:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764065.342534:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764065.342534:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876de00.
00000100:00000001:2.0:1787764065.342537:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764065.342538:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764065.342539:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764065.342540:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832687872/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764090 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764065.342544:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764065.342545:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764065.342546:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764065.342548:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764065.342548:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764065.342549:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764065.342551:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764065.342551:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014800.
02000000:00000001:2.0:1787764065.342553:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764065.342553:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764065.342554:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:2.0:1787764065.342555:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764065.342556:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764065.342557:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764065.342560:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764065.342561:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764065.342562:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764065.342563:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764065.980199:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764065.980205:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764065.980207:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764065.980212:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764065.980215:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764065.980219:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764065.980222:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764065.980222:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764065.980225:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764065.980226:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764065.980231:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764065.980233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764065.980237:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764065.980241:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764065.980242:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764065.980277:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764065.980281:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764065.980284:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764066.364275:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764066.364285:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764066.364288:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764066.364295:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764066.364300:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764066.364304:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764066.364307:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764066.364310:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764066.364314:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764066.620212:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316453620, at 4316453632
00000100:00000010:3.0:1787764066.620247:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7240.
00000100:00000040:2.0:1787764066.620288:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316453620, at 4316453632
00000100:00000010:2.0:1787764066.620301:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7720.
00000100:00000001:1.0:1787764067.004256:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764067.004263:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764067.004266:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764067.004271:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764067.004274:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764067.004278:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764067.004280:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764067.004283:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764067.004283:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764067.004288:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764067.004294:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764067.004303:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764067.004305:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764067.004312:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764067.004314:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764067.004359:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764067.004363:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764067.004366:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764067.388268:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764067.388279:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764067.388281:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764067.388289:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764067.388294:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764067.388298:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764067.388301:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764067.388305:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764067.388309:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764068.028200:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764068.028202:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764068.028206:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764068.028211:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764068.028212:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764068.028216:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764068.028217:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764068.028221:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764068.028224:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764068.028226:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764068.028228:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764068.028230:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764068.028231:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764068.028234:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764068.028238:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764068.028239:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764068.028242:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764068.028244:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764068.028249:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764068.028252:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764068.028255:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764068.028336:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764068.028339:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764068.028342:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764068.412254:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764068.412263:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764068.412266:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764068.412272:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764068.412277:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764068.412281:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764068.412283:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764068.412286:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764068.412289:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764069.052249:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764069.052257:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764069.052264:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764069.052264:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764069.052267:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764069.052270:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764069.052273:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764069.052274:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764069.052277:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764069.052278:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764069.052283:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764069.052289:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764069.052292:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764069.052294:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764069.052298:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764069.052336:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764069.052340:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764069.052343:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764069.436261:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764069.436270:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764069.436271:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764069.436281:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764069.436288:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764069.436296:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764069.436302:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764069.436306:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764069.436311:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764070.076219:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764070.076221:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764070.076228:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764070.076228:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764070.076231:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764070.076237:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764070.076240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764070.076241:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764070.076245:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764070.076246:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.076252:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764070.076255:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764070.076255:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764070.076258:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764070.076259:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764070.076286:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764070.076289:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764070.076291:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787764070.460300:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764070.460308:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787764070.460309:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787764070.460311:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787764070.460315:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1787764070.460316:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764070.460319:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764070.460324:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00100000:2.0:1787764070.460325:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764070.460330:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764070.460335:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787764070.460337:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21785
00000100:00000001:2.0:1787764070.460342:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000400:00000200:0.0:1787764070.460347:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764070.460348:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.460353:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787764070.460354:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764070.460359:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764070.460362:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764070.460364:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764070.460366:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764070.460368:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764070.460371:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764070.460373:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764070.460375:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764070.460378:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764070.460380:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764070.460382:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787764070.460384:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764070.460387:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764070.460388:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764070.460390:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764070.460393:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787764070.460395:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764070.460397:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764070.460399:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764070.460402:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764070.460404:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764070.460406:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764070.460408:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764070.460415:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787764070.460418:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764070.460421:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764070.460422:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764070.460424:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764070.460427:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764070.460430:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764070.460436:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787764070.460441:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764070.460449:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832688000/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764070.460457:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764070.460460:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764070.460464:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832688000/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764070.460481:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764070.460483:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764070.460485:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764070.460486:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764070.460488:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764070.460490:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764070.460493:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21785
00000400:00000200:0.0:1787764070.460496:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764070.460499:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764070.460500:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764070.460503:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764070.460505:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764070.460506:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764070.460509:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764070.460509:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1787764070.460510:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764070.460514:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787764070.460515:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1787764070.460516:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000040:0.0:1787764070.460518:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787764070.460521:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787764070.460522:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1787764070.460523:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:2.0:1787764070.460531:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832688000/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764070.460535:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764070.460539:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764070.460543:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764070.460548:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764070.460550:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:0.0:1787764070.460551:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00000001:2.0:1787764070.460554:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764070.460556:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764070.460560:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764070.460560:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787764070.460564:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764070.460565:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764070.460569:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764070.460569:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764070.460574:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764070.460574:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832688000:192.168.203.110@tcp:250:kworker.0
00000100:00000001:0.0:1787764070.460576:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764070.460581:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:0.0:1787764070.460582:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787764070.460585:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764070.460587:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764070.460587:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:0.0:1787764070.460590:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
02000000:00000001:2.0:1787764070.460591:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787764070.460595:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787764070.460597:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764070.460599:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764070.460600:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764070.460603:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764070.460604:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1787764070.460605:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764070.460610:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787764070.460612:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764070.460615:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
00000100:00100000:0.0:1787764070.460617:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832688128/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1787764070.460621:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764070.460628:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764070.460631:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764070.460633:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:0.0:1787764070.460637:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832688128/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1787764070.460641:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000100:00000001:0.0:1787764070.460645:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764070.460648:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764070.460650:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787764070.460652:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:2.0:1787764070.460652:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00080000:0.0:1787764070.460654:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764070.460658:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764070.460660:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21785
00000100:00000200:2.0:1787764070.460662:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832688000, portal 25
00000400:00000200:0.0:1787764070.460663:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764070.460668:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787764070.460669:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00080000:0.0:1787764070.460671:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787764070.460672:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000100:00000040:0.0:1787764070.460675:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764070.460677:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764070.460683:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000040:2.0:1787764070.460683:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832688000/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764095 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764070.460685:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764070.460690:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764070.460692:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787764070.460695:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764070.460697:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787764070.460703:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000001:2.0:1787764070.460703:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000001:0.0:1787764070.460707:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:2.0:1787764070.460711:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787764070.460714:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000010:2.0:1787764070.460716:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000001:0.0:1787764070.460717:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764070.460721:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000200:2.0:1787764070.460721:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832688000, offset 0
00000100:00000001:0.0:1787764070.460726:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764070.460728:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787764070.460733:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:2.0:1787764070.460734:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787764070.460736:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764070.460739:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764070.460754:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764070.460756:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764070.460758:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764070.460762:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:2.0:1787764070.460762:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000008:00000001:0.0:1787764070.460766:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764070.460768:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764070.460772:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787764070.460776:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000400:00000200:2.0:1787764070.460776:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
02000000:00000001:0.0:1787764070.460781:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764070.460783:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764070.460785:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764070.460787:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1787764070.460787:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:0.0:1787764070.460789:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764070.460793:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000400:00000200:2.0:1787764070.460796:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:0.0:1787764070.460798:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764070.460802:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00100000:0.0:1787764070.460807:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832688256/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787764070.460807:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764070.460816:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58610:58610:256:4294967295] 192.168.203.110@tcp LPNI seq info [58610:58610:8:4294967295]
00000100:00000001:0.0:1787764070.460823:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764070.460826:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764070.460827:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:0.0:1787764070.460831:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832688256/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764070.460843:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1787764070.460848:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764070.460852:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:2.0:1787764070.460852:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787764070.460859:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000010:2.0:1787764070.460861:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000100:00080000:0.0:1787764070.460862:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764070.460867:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787764070.460868:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764070.460869:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:2.0:1787764070.460874:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787764070.460875:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:2.0:1787764070.460880:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000100:00080000:0.0:1787764070.460881:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21785
00000400:00000200:0.0:1787764070.460888:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:2.1:1787764070.460892:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000400:00000200:0.0:1787764070.460894:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000800:00000200:2.1:1787764070.460897:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000100:00080000:0.0:1787764070.460898:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764070.460904:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764070.460907:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000100:2.0:1787764070.460908:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787764070.460912:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000010:2.0:1787764070.460913:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000100:00000001:0.0:1787764070.460916:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764070.460918:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764070.460919:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000400:00000200:2.0:1787764070.460920:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764070.460926:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:0.0:1787764070.460928:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764070.460932:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000400:00000200:2.0:1787764070.460932:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000020:00000001:0.0:1787764070.460937:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000010:2.0:1787764070.460937:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000020:00000001:0.0:1787764070.460943:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:2.0:1787764070.460944:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:0.0:1787764070.460947:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764070.460950:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:2.0:1787764070.460952:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e1c0 x1874599832688000/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764095 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764070.460954:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764070.460958:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764070.460963:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787764070.460968:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764070.460971:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764070.460971:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:0.0:1787764070.460975:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764070.460979:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1787764070.460979:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832688000/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764095 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787764070.460984:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764070.460989:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764070.460993:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:2.0:1787764070.460994:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1787764070.460997:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764070.460998:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787764070.461000:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764070.461004:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000100:00000001:2.0:1787764070.461005:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787764070.461006:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787764070.461011:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764070.461013:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764070.461014:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764070.461015:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764070.461018:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787764070.461018:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316458739
02000000:00000001:0.0:1787764070.461021:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764070.461026:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7720.
02000000:00000010:0.0:1787764070.461027:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787764070.461032:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764070.461035:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00100000:0.0:1787764070.461036:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832688384/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764070.461043:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764070.461043:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316458739
00000100:00000001:0.0:1787764070.461048:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764070.461050:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7180.
00000100:00000040:0.0:1787764070.461051:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832688384/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461055:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787764070.461060:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764070.461062:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764070.461064:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764070.461066:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21795)
00000100:00000001:2.0:1787764070.461067:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764070.461075:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764070/real 1787764070]  req@ffff9751f876e1c0 x1874599832688000/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764070.461090:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764070.461092:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764070.461093:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764070.461096:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764070.461097:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764070.461099:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764070.461103:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832688000/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764070.461120:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832688000/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461129:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764070.461132:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.461180:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764070.461183:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764070.461188:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832688000/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461193:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764070.461197:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832688000/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461201:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764070.461203:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.461204:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764070.461205:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.461207:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764070.461208:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764070.461210:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764070.461213:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764070.461215:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764070.461217:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832688000/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764070.461223:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832688000:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764070.461226:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.461227:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764070.461229:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832688000/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461237:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764070.461239:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764070.461241:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764070.461244:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764070.461245:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764070.461246:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764070.461248:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764070.461250:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
02000000:00000001:2.0:1787764070.461253:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764070.461255:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764070.461256:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:2.0:1787764070.461259:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764070.461260:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.461262:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.461267:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764070.461268:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764070.461269:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764070.461271:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764070.461272:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764070.461274:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832688128/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461279:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764070.461280:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764070.461281:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764070.461283:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764070.461284:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764070.461286:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832688128:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764070.461288:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764070.461290:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764070.461291:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764070.461293:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764070.461295:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764070.461296:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764070.461299:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764070.461302:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764070.461304:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764070.461307:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832688128, portal 10
00000100:00000001:2.0:1787764070.461308:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764070.461309:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00000040:2.0:1787764070.461314:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832688128/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764095 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461319:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764070.461321:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764070.461322:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764070.461324:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832688128, offset 0
00000400:00000200:2.0:1787764070.461326:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764070.461329:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764070.461332:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764070.461334:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764070.461337:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764070.461340:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764070.461342:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764070.461346:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58611:58611:256:4294967295] 192.168.203.110@tcp LPNI seq info [58611:58611:8:4294967295]
00000400:00000200:2.0:1787764070.461350:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764070.461356:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764070.461361:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764070.461365:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764070.461369:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764070.461373:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787764070.461377:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764070.461380:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1787764070.461385:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764070.461387:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764070.461390:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764070.461392:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764070.461395:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764070.461396:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764070.461399:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764070.461402:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e580 x1874599832688128/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764095 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461407:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764070.461409:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832688128/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764095 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461414:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764070.461415:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764070.461418:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764070.461421:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764070.461423:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764070.461424:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316457504
00000100:00000001:2.0:1787764070.461426:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316458739
00000100:00000001:2.0:1787764070.461428:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316457504
00000100:00000001:2.0:1787764070.461429:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316458739
00000100:00000001:2.0:1787764070.461431:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764070.461432:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764070.461433:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764070.461435:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764070/real 1787764070]  req@ffff9751f876e580 x1874599832688128/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764070.461440:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764070.461441:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764070.461442:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764070.461444:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764070.461445:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764070.461447:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764070.461449:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832688128/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764070.461454:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832688128/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461462:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764070.461463:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.461464:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764070.461465:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764070.461467:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832688128/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461472:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764070.461475:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832688128/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461482:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764070.461485:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.461486:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764070.461487:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.461489:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764070.461501:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764070.461505:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764070.461508:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764070.461511:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764070.461516:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832688128/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764070.461525:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832688128:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764070.461528:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764070.461530:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f0c0 x1874599832688256/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461535:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764070.461536:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764070.461538:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764070.461539:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764070.461540:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764070.461542:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832688256:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764070.461545:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764070.461546:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764070.461547:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764070.461548:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764070.461551:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764070.461552:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764070.461555:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764070.461556:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764070.461559:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764070.461561:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832688256, portal 10
00000100:00000001:2.0:1787764070.461562:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764070.461563:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049530560 : -115096660021056 : ffff9751f876f0c0)
00000100:00000040:2.0:1787764070.461567:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f0c0 x1874599832688256/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764095 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461572:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764070.461573:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764070.461575:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764070.461576:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832688256, offset 0
00000400:00000200:2.0:1787764070.461579:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764070.461581:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764070.461584:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764070.461586:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764070.461588:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764070.461590:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764070.461592:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764070.461594:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58612:58612:256:4294967295] 192.168.203.110@tcp LPNI seq info [58612:58612:8:4294967295]
00000400:00000200:2.0:1787764070.461597:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764070.461601:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764070.461606:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764070.461609:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764070.461614:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764070.461616:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787764070.461621:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764070.461624:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1787764070.461628:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764070.461630:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764070.461633:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764070.461637:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764070.461640:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764070.461642:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764070.461645:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764070.461649:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f0c0 x1874599832688256/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764095 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461656:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764070.461658:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f0c0 x1874599832688256/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764095 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461662:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764070.461663:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764070.461666:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764070.461668:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764070.461670:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764070.461672:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316457492
00000100:00000001:2.0:1787764070.461674:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316458739
00000100:00000001:2.0:1787764070.461675:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316457492
00000100:00000001:2.0:1787764070.461677:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316458739
00000100:00000001:2.0:1787764070.461678:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764070.461679:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764070.461680:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764070.461683:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764070/real 1787764070]  req@ffff9751f876f0c0 x1874599832688256/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764070.461689:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764070.461690:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764070.461691:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764070.461692:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764070.461693:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764070.461694:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764070.461696:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f0c0 x1874599832688256/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764070.461702:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f0c0 x1874599832688256/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461705:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764070.461706:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.461707:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764070.461708:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764070.461710:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f0c0 x1874599832688256/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461714:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764070.461717:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f0c0 x1874599832688256/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461722:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764070.461723:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.461724:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764070.461725:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.461726:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764070.461727:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764070.461729:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764070.461731:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764070.461733:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764070.461737:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f0c0 x1874599832688256/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764070.461759:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832688256:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764070.461763:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764070.461767:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832688384/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461775:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764070.461776:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764070.461779:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764070.461781:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764070.461783:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764070.461786:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832688384:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764070.461790:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764070.461792:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764070.461793:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764070.461795:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764070.461798:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764070.461800:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764070.461802:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764070.461804:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764070.461807:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764070.461809:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832688384, portal 4
00000100:00000001:2.0:1787764070.461811:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764070.461812:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
00000100:00000040:2.0:1787764070.461816:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832688384/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764095 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461821:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764070.461822:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764070.461823:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764070.461825:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832688384, offset 0
00000400:00000200:2.0:1787764070.461828:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764070.461831:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764070.461833:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764070.461835:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764070.461837:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764070.461839:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764070.461841:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764070.461843:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58613:58613:256:4294967295] 192.168.203.110@tcp LPNI seq info [58613:58613:8:4294967295]
00000400:00000200:2.0:1787764070.461846:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764070.461850:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764070.461853:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764070.461855:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764070.461857:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764070.461859:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:2.1:1787764070.461862:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764070.461865:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:2.0:1787764070.461867:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764070.461868:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764070.461870:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764070.461873:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764070.461875:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764070.461876:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764070.461877:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764070.461880:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876ed00 x1874599832688384/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764095 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461885:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764070.461887:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832688384/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764095 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461891:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764070.461892:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764070.461895:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764070.461897:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764070.461899:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764070.461901:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316457509
00000100:00000001:2.0:1787764070.461902:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316458739
00000100:00000001:2.0:1787764070.461904:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316457509
00000100:00000001:2.0:1787764070.461905:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316458739
00000100:00000001:2.0:1787764070.461907:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764070.461908:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764070.461909:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764070.461911:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764070/real 1787764070]  req@ffff9751f876ed00 x1874599832688384/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764070.461916:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764070.461917:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764070.461918:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764070.461919:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764070.461921:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764070.461922:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764070.461924:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832688384/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764070.461929:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832688384/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461933:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764070.461934:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.461935:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764070.461936:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764070.461938:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832688384/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461942:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764070.461945:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832688384/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461949:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764070.461951:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.461952:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764070.461953:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.461954:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764070.461955:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764070.461957:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764070.461959:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764070.461960:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764070.461962:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832688384/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764070.461967:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832688384:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764070.461969:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.461971:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764070.461972:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832688128/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.461976:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764070.461977:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764070.461979:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764070.461980:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764070.461981:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764070.461982:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764070.461984:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764070.461985:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:2.0:1787764070.461987:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764070.461988:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764070.461989:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:2.0:1787764070.461991:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764070.461991:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.461993:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764070.461994:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f0c0 x1874599832688256/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.462003:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764070.462004:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764070.462005:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764070.462006:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764070.462007:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764070.462008:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764070.462009:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764070.462010:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016c00.
02000000:00000001:2.0:1787764070.462011:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764070.462012:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764070.462013:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f0c0.
00000100:00000001:2.0:1787764070.462014:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764070.462015:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.462016:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764070.462017:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832688384/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764095 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764070.462021:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764070.462022:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764070.462022:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764070.462024:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764070.462024:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764070.462025:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764070.462026:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764070.462027:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017800.
02000000:00000001:2.0:1787764070.462029:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764070.462030:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764070.462030:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:2.0:1787764070.462032:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764070.462032:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.462033:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764070.462036:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764070.462037:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764070.462038:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764070.462039:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764071.100222:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764071.100226:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764071.100229:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764071.100231:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764071.100232:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764071.100238:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764071.100239:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764071.100241:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764071.100243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764071.100244:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764071.100250:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764071.100253:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764071.100256:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764071.100258:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764071.100262:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764071.100302:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764071.100306:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764071.100309:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764071.484215:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764071.484222:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764071.484224:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764071.484229:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764071.484233:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764071.484236:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764071.484239:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764071.484242:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764071.484245:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764071.740183:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316458739, at 4316458752
00000100:00000010:3.0:1787764071.740196:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7180.
00000100:00000040:2.0:1787764071.740240:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316458739, at 4316458752
00000100:00000010:2.0:1787764071.740248:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7720.
00000100:00000001:2.0:1787764072.124226:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764072.124226:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764072.124233:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764072.124233:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764072.124235:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764072.124240:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764072.124242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764072.124243:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764072.124247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764072.124248:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764072.124253:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764072.124255:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764072.124256:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764072.124261:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764072.124262:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764072.124266:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764072.124270:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764072.124273:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764072.124277:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764072.124279:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764072.124281:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764072.124340:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764072.124344:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764072.124346:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764072.508249:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764072.508257:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764072.508259:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764072.508263:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764072.508267:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764072.508270:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764072.508272:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764072.508275:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764072.508277:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764073.148233:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764073.148240:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764073.148243:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764073.148247:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764073.148247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764073.148251:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764073.148253:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764073.148257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764073.148258:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764073.148261:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764073.148262:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764073.148265:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764073.148266:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764073.148268:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764073.148269:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764073.148336:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764073.148340:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764073.148342:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764073.532230:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764073.532237:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764073.532239:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764073.532243:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764073.532254:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764073.532257:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764073.532259:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764073.532261:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764073.532263:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764074.172238:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764074.172245:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764074.172246:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764074.172248:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764074.172251:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764074.172254:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764074.172258:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764074.172259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764074.172261:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764074.172263:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764074.172264:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764074.172268:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764074.172271:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764074.172272:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764074.172275:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764074.172332:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764074.172336:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764074.172339:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764074.556282:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764074.556290:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764074.556293:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764074.556299:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764074.556303:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764074.556307:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764074.556310:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764074.556313:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764074.556316:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764075.196253:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764075.196262:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764075.196264:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764075.196266:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764075.196271:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00100000:2.0:1787764075.196273:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764075.196276:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764075.196279:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764075.196281:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764075.196284:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764075.196288:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764075.196289:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764075.196292:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764075.196294:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764075.196297:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764075.196300:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764075.196302:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764075.196309:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764075.196311:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764075.196315:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764075.196317:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764075.196367:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764075.196371:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764075.196374:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764075.580199:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764075.580206:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764075.580208:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764075.580213:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764075.580216:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764075.580219:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787764075.580221:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764075.580222:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764075.580224:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764075.580229:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764075.580231:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787764075.580235:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764075.580239:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764075.580244:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764075.580251:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21790
00000400:00000200:0.0:1787764075.580259:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764075.580266:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764075.580269:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764075.580273:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764075.580276:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764075.580278:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764075.580281:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764075.580285:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764075.580288:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764075.580291:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764075.580295:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764075.580297:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764075.580300:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787764075.580303:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764075.580305:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764075.580307:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764075.580309:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764075.580312:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787764075.580315:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764075.580318:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764075.580321:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764075.580325:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764075.580328:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764075.580330:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764075.580332:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764075.580340:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787764075.580344:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764075.580348:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764075.580350:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764075.580353:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764075.580356:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764075.580360:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764075.580367:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787764075.580373:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764075.580381:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832688512/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764075.580391:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764075.580395:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764075.580400:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832688512/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764075.580421:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764075.580424:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764075.580427:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764075.580429:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764075.580431:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764075.580434:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764075.580437:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21790
00000400:00000200:0.0:1787764075.580441:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764075.580443:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:0.0:1787764075.580446:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787764075.580448:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1787764075.580450:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787764075.580451:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787764075.580457:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764075.580461:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764075.580461:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e940 x1874599832688512/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764075.580468:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764075.580474:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764075.580474:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764075.580476:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764075.580479:0:304484:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:2.0:1787764075.580481:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:0.0:1787764075.580482:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
02000000:00000001:2.0:1787764075.580485:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:0.0:1787764075.580486:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:2.0:1787764075.580489:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787764075.580491:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00100000:2.0:1787764075.580494:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832688512:192.168.203.110@tcp:250:kworker.0
00000020:00000001:0.0:1787764075.580496:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787764075.580499:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:0.0:1787764075.580501:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:2.0:1787764075.580504:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764075.580507:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764075.580509:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764075.580510:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1787764075.580515:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000010:2.0:1787764075.580517:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
00000100:00000040:0.0:1787764075.580518:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764075.580521:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764075.580524:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787764075.580529:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:0.0:1787764075.580530:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000400:00000010:2.0:1787764075.580534:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000100:00080000:0.0:1787764075.580536:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764075.580540:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764075.580541:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
02000000:00000001:0.0:1787764075.580543:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:2.0:1787764075.580549:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832688512, portal 25
02000000:00000001:0.0:1787764075.580551:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764075.580553:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:0.0:1787764075.580557:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764075.580557:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049528640 : -115096660022976 : ffff9751f876e940)
00000100:00000001:0.0:1787764075.580563:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:2.0:1787764075.580567:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e940 x1874599832688512/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764100 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1787764075.580570:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787764075.580577:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764075.580577:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1787764075.580582:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764075.580582:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764075.580589:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000010:0.0:1787764075.580590:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
00000100:00000200:2.0:1787764075.580591:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832688512, offset 0
00000020:00000040:0.0:1787764075.580595:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000200:2.0:1787764075.580601:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
02000000:00000001:0.0:1787764075.580602:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764075.580605:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764075.580607:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764075.580610:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1787764075.580610:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787764075.580614:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764075.580619:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
02000000:00000010:0.0:1787764075.580622:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787764075.580627:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764075.580631:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:0.0:1787764075.580636:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832688640/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787764075.580639:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764075.580645:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764075.580652:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:0.0:1787764075.580657:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1787764075.580657:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58614:58614:256:4294967295] 192.168.203.110@tcp LPNI seq info [58614:58614:8:4294967295]
00000100:00000001:0.0:1787764075.580663:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764075.580665:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:0.0:1787764075.580672:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832688640/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764075.580676:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764075.580688:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764075.580695:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000100:00000001:0.0:1787764075.580696:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764075.580701:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:2.0:1787764075.580720:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00080000:0.0:1787764075.580722:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787764075.580726:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:2.0:1787764075.580728:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000100:00080000:0.0:1787764075.580729:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764075.580736:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:2.1:1787764075.580737:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00080000:0.0:1787764075.580743:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21790
00000800:00000200:2.1:1787764075.580743:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000400:00000200:0.0:1787764075.580753:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000100:2.0:1787764075.580754:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764075.580757:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:0.0:1787764075.580762:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1787764075.580764:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:0.0:1787764075.580766:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1787764075.580771:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:0.0:1787764075.580774:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764075.580777:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000100:00000001:0.0:1787764075.580779:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1787764075.580780:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000040:0.0:1787764075.580785:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764075.580786:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1787764075.580791:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000200:2.0:1787764075.580792:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e940 x1874599832688512/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764100 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787764075.580796:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764075.580799:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787764075.580803:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764075.580806:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787764075.580811:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000001:2.0:1787764075.580812:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000001:0.0:1787764075.580815:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:2.0:1787764075.580819:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e940 x1874599832688512/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764100 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764075.580822:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764075.580828:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764075.580832:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764075.580838:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764075.580841:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787764075.580843:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00000001:2.0:1787764075.580845:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:0.0:1787764075.580851:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764075.580854:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764075.580854:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764075.580857:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1787764075.580861:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787764075.580863:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764075.580864:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316463859
02000000:00000001:0.0:1787764075.580869:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764075.580872:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7720.
00000100:00000001:0.0:1787764075.580874:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764075.580878:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787764075.580881:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764075.580883:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764075.580888:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000100:00000001:2.0:1787764075.580888:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000020:00000040:0.0:1787764075.580893:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000001:2.0:1787764075.580897:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316463859
02000000:00000001:0.0:1787764075.580898:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764075.580902:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764075.580903:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb72a0.
00000100:00000001:0.0:1787764075.580905:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1787764075.580908:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
02000000:00000001:0.0:1787764075.580909:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787764075.580912:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764075.580913:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764075.580916:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764075.580919:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000010:0.0:1787764075.580920:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00100000:2.0:1787764075.580924:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764075/real 1787764075]  req@ffff9751f876e940 x1874599832688512/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764075.580926:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:2.0:1787764075.580935:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00100000:0.0:1787764075.580937:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832688768/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000001:2.0:1787764075.580938:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764075.580941:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764075.580945:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764075.580948:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764075.580952:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:0.0:1787764075.580956:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:2.0:1787764075.580959:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e940 x1874599832688512/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764075.580961:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764075.580970:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832688768/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764075.580972:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e940 x1874599832688512/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.580990:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764075.580993:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764075.580995:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764075.580996:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764075.580998:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787764075.581001:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1787764075.581002:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e940 x1874599832688512/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764075.581011:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1787764075.581014:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787764075.581018:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1787764075.581020:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e940 x1874599832688512/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787764075.581027:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787764075.581032:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787764075.581033:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787764075.581037:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764075.581037:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764075.581040:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764075.581042:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764075.581044:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787764075.581045:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764075.581049:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787764075.581052:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21790
00000100:00080000:2.0:1787764075.581053:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764075.581059:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000400:00000200:0.0:1787764075.581062:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764075.581063:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000400:00000200:0.0:1787764075.581071:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764075.581075:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764075.581077:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e940 x1874599832688512/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764075.581082:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764075.581089:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:2.0:1787764075.581090:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832688512:192.168.203.110@tcp:250:kworker.0
00000100:00000040:0.0:1787764075.581095:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764075.581097:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764075.581101:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764075.581102:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764075.581106:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e940 x1874599832688512/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787764075.581108:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764075.581111:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787764075.581113:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764075.581116:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787764075.581121:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:2.0:1787764075.581124:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
00000020:00000001:0.0:1787764075.581126:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
02000000:00000001:2.0:1787764075.581128:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764075.581131:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764075.581151:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764075.581153:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764075.581155:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764075.581156:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764075.581158:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016c00.
00000100:00000040:0.0:1787764075.581162:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764075.581162:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764075.581165:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764075.581167:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e940.
00000100:00000001:0.0:1787764075.581169:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764075.581172:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000040:0.0:1787764075.581174:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764075.581174:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764075.581178:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764075.581180:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764075.581185:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:2.0:1787764075.581185:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764075.581188:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764075.581192:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:0.0:1787764075.581193:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764075.581194:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764075.581197:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1787764075.581199:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764075.581202:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f840 x1874599832688640/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787764075.581204:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764075.581210:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764075.581210:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764075.581213:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764075.581216:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764075.581216:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764075.581220:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764075.581221:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1787764075.581223:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1787764075.581227:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00100000:2.0:1787764075.581227:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832688640:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764075.581233:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:0.0:1787764075.581235:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764075.581236:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764075.581238:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764075.581241:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764075.581241:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764075.581245:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764075.581248:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764075.581249:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000400:00000200:2.0:1787764075.581252:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:0.0:1787764075.581255:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000400:00000010:2.0:1787764075.581257:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
02000000:00000001:0.0:1787764075.581261:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:2.0:1787764075.581262:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
02000000:00000001:0.0:1787764075.581265:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764075.581266:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832688640, portal 10
00000100:00000001:0.0:1787764075.581270:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1787764075.581270:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764075.581273:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049532480 : -115096660019136 : ffff9751f876f840)
02000000:00000001:0.0:1787764075.581275:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764075.581279:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764075.581280:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f840 x1874599832688640/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764100 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1787764075.581287:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:2.0:1787764075.581287:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764075.581292:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787764075.581294:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764075.581295:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00100000:0.0:1787764075.581303:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832688896/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:2.0:1787764075.581305:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832688640, offset 0
00000400:00000200:2.0:1787764075.581315:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787764075.581322:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1787764075.581322:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764075.581327:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:0.0:1787764075.581328:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764075.581332:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764075.581337:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000040:0.0:1787764075.581338:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832688896/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764075.581339:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764075.581343:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764075.581348:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58615:58615:256:4294967295] 192.168.203.110@tcp LPNI seq info [58615:58615:8:4294967295]
00000400:00000200:2.0:1787764075.581359:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764075.581367:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764075.581369:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:0.0:1787764075.581373:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21800)
00000800:00000200:2.0:1787764075.581374:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764075.581380:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764075.581383:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764075.581386:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787764075.581389:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764075.581392:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1787764075.581395:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764075.581397:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764075.581399:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764075.581402:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764075.581404:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764075.581405:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764075.581407:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764075.581409:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f840 x1874599832688640/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764100 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581415:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764075.581417:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f840 x1874599832688640/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764100 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581426:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764075.581427:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764075.581429:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764075.581431:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764075.581433:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764075.581435:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316462624
00000100:00000001:2.0:1787764075.581437:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316463859
00000100:00000001:2.0:1787764075.581438:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316462624
00000100:00000001:2.0:1787764075.581440:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316463859
00000100:00000001:2.0:1787764075.581441:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764075.581443:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764075.581444:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764075.581446:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764075/real 1787764075]  req@ffff9751f876f840 x1874599832688640/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764075.581451:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764075.581452:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764075.581453:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764075.581454:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764075.581455:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764075.581456:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764075.581458:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f840 x1874599832688640/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764075.581463:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f840 x1874599832688640/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581467:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764075.581468:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764075.581469:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764075.581470:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764075.581472:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f840 x1874599832688640/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581477:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764075.581480:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f840 x1874599832688640/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581485:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764075.581486:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764075.581487:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764075.581488:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764075.581489:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764075.581498:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764075.581500:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764075.581502:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764075.581503:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764075.581505:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f840 x1874599832688640/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764075.581510:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832688640:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764075.581513:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764075.581515:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cf00 x1874599832688768/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581520:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764075.581521:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764075.581522:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764075.581523:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764075.581524:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764075.581526:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832688768:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764075.581528:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764075.581529:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764075.581531:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764075.581532:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764075.581534:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764075.581536:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764075.581538:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764075.581539:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764075.581541:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764075.581543:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832688768, portal 10
00000100:00000001:2.0:1787764075.581545:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764075.581546:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049521920 : -115096660029696 : ffff9751f876cf00)
00000100:00000040:2.0:1787764075.581549:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cf00 x1874599832688768/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764100 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581554:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764075.581556:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764075.581557:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764075.581559:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832688768, offset 0
00000400:00000200:2.0:1787764075.581561:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764075.581563:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764075.581565:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764075.581567:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764075.581568:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764075.581570:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764075.581572:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764075.581574:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58616:58616:256:4294967295] 192.168.203.110@tcp LPNI seq info [58616:58616:8:4294967295]
00000400:00000200:2.0:1787764075.581577:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764075.581581:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764075.581584:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764075.581586:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764075.581589:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764075.581591:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787764075.581593:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764075.581595:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1787764075.581598:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764075.581600:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764075.581602:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764075.581604:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764075.581606:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764075.581607:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764075.581609:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764075.581611:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cf00 x1874599832688768/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764100 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581615:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764075.581617:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cf00 x1874599832688768/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764100 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581621:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764075.581622:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764075.581623:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764075.581625:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764075.581627:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764075.581628:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316462612
00000100:00000001:2.0:1787764075.581630:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316463859
00000100:00000001:2.0:1787764075.581632:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316462612
00000100:00000001:2.0:1787764075.581633:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316463859
00000100:00000001:2.0:1787764075.581634:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764075.581635:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764075.581636:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764075.581639:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764075/real 1787764075]  req@ffff9751f876cf00 x1874599832688768/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764075.581643:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764075.581644:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764075.581645:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764075.581646:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764075.581647:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764075.581649:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764075.581650:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cf00 x1874599832688768/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764075.581656:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cf00 x1874599832688768/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581660:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764075.581661:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764075.581662:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764075.581662:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764075.581664:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cf00 x1874599832688768/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581668:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764075.581671:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cf00 x1874599832688768/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581675:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764075.581676:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764075.581677:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764075.581678:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764075.581679:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764075.581680:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764075.581681:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764075.581683:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764075.581684:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764075.581686:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cf00 x1874599832688768/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764075.581691:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832688768:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764075.581694:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764075.581696:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764075.581697:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f840 x1874599832688640/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581702:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764075.581703:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764075.581717:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764075.581721:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764075.581722:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764075.581723:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764075.581724:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764075.581725:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:2.0:1787764075.581727:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764075.581728:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764075.581729:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f840.
00000100:00000001:2.0:1787764075.581731:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764075.581732:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764075.581733:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764075.581736:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cf00 x1874599832688768/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581740:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764075.581741:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764075.581742:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764075.581743:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764075.581744:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764075.581745:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764075.581746:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764075.581747:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
02000000:00000001:2.0:1787764075.581749:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764075.581749:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764075.581750:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cf00.
00000100:00000001:2.0:1787764075.581752:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764075.581753:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764075.581754:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764075.581756:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764075.581757:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764075.581758:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764075.581760:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764075.581761:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764075.581763:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e940 x1874599832688896/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581768:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764075.581769:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764075.581770:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764075.581771:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764075.581773:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764075.581775:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832688896:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764075.581777:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764075.581778:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764075.581779:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764075.581780:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764075.581782:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764075.581784:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764075.581786:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764075.581788:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764075.581790:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764075.581792:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832688896, portal 4
00000100:00000001:2.0:1787764075.581794:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764075.581794:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049528640 : -115096660022976 : ffff9751f876e940)
00000100:00000040:2.0:1787764075.581800:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e940 x1874599832688896/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764100 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581807:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764075.581809:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764075.581812:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764075.581814:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832688896, offset 0
00000400:00000200:2.0:1787764075.581818:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764075.581822:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764075.581826:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764075.581829:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764075.581831:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764075.581834:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764075.581837:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764075.581840:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58617:58617:256:4294967295] 192.168.203.110@tcp LPNI seq info [58617:58617:8:4294967295]
00000400:00000200:2.0:1787764075.581845:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764075.581851:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764075.581856:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764075.581859:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764075.581861:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764075.581863:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:2.1:1787764075.581866:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764075.581868:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:2.0:1787764075.581871:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764075.581872:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764075.581874:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764075.581876:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764075.581879:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764075.581880:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764075.581882:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764075.581884:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e940 x1874599832688896/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764100 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581890:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764075.581892:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e940 x1874599832688896/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764100 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581896:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764075.581897:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764075.581899:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764075.581901:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764075.581903:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764075.581905:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316462629
00000100:00000001:2.0:1787764075.581906:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316463859
00000100:00000001:2.0:1787764075.581908:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316462629
00000100:00000001:2.0:1787764075.581910:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316463859
00000100:00000001:2.0:1787764075.581912:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764075.581913:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764075.581915:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764075.581917:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764075/real 1787764075]  req@ffff9751f876e940 x1874599832688896/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764075.581922:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764075.581923:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764075.581924:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764075.581925:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764075.581926:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764075.581928:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764075.581929:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e940 x1874599832688896/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764075.581935:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e940 x1874599832688896/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581939:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764075.581940:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764075.581941:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764075.581942:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764075.581944:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e940 x1874599832688896/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581948:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764075.581950:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e940 x1874599832688896/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581954:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764075.581955:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764075.581956:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764075.581957:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764075.581959:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764075.581960:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764075.581962:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764075.581965:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764075.581966:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764075.581968:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e940 x1874599832688896/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764075.581973:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832688896:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764075.581975:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764075.581976:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764075.581978:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e940 x1874599832688896/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764100 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764075.581982:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764075.581983:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764075.581984:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764075.581985:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764075.581986:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764075.581987:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764075.581988:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764075.581989:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014400.
02000000:00000001:2.0:1787764075.581991:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764075.581992:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764075.581993:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e940.
00000100:00000001:2.0:1787764075.581994:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764075.581995:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764075.581997:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764075.581999:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764075.582000:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764075.582001:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764075.582002:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764076.220238:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764076.220246:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764076.220246:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764076.220248:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764076.220252:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764076.220254:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764076.220258:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764076.220259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764076.220262:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764076.220262:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764076.220266:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764076.220269:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764076.220270:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764076.220272:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764076.220273:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764076.220330:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764076.220335:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764076.220339:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764076.604253:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764076.604260:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764076.604261:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764076.604267:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764076.604270:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764076.604273:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764076.604275:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764076.604278:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764076.604281:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764076.860252:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316463859, at 4316463872
00000100:00000010:3.0:1787764076.860268:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb72a0.
00000100:00000040:2.0:1787764076.860274:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316463859, at 4316463872
00000100:00000010:2.0:1787764076.860284:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7720.
00000100:00000001:2.0:1787764077.244230:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764077.244233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764077.244237:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764077.244239:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764077.244241:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764077.244245:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764077.244250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764077.244251:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764077.244253:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764077.244257:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764077.244258:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764077.244260:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764077.244264:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764077.244269:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764077.244272:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764077.244314:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764077.244317:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764077.244320:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764077.628208:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764077.628214:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764077.628216:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764077.628221:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764077.628224:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764077.628226:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764077.628228:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764077.628231:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764077.628233:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764078.268242:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764078.268247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764078.268251:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764078.268253:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764078.268256:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764078.268262:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764078.268266:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764078.268270:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764078.268273:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764078.268274:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764078.268278:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764078.268280:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764078.268282:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764078.268284:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764078.268290:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764078.268293:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764078.268294:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764078.268296:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764078.268299:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764078.268303:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764078.268305:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764078.268367:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764078.268371:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764078.268374:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764078.652240:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764078.652249:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764078.652252:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764078.652257:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764078.652262:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764078.652267:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764078.652270:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764078.652272:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764078.652275:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764079.292190:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764079.292196:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764079.292198:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764079.292198:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764079.292203:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764079.292204:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764079.292210:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764079.292210:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764079.292213:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764079.292214:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764079.292217:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764079.292218:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764079.292219:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764079.292222:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764079.292225:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764079.292282:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764079.292287:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764079.292291:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764079.676249:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764079.676258:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764079.676261:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764079.676271:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764079.676276:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764079.676279:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764079.676282:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764079.676285:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764079.676288:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764080.316232:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764080.316232:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764080.316239:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764080.316241:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764080.316245:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764080.316249:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764080.316253:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764080.316257:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764080.316258:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764080.316261:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764080.316265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764080.316266:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.316274:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764080.316274:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764080.316278:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764080.316339:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764080.316344:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764080.316347:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764080.700243:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764080.700262:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764080.700264:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764080.700268:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764080.700272:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1787764080.700275:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764080.700276:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.700279:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:0.0:1787764080.700283:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787764080.700283:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.700287:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764080.700288:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764080.700293:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764080.700296:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764080.700302:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21795
00000400:00000200:0.0:1787764080.700309:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764080.700314:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764080.700317:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764080.700319:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764080.700322:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764080.700325:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764080.700327:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764080.700330:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764080.700332:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764080.700334:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764080.700337:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764080.700339:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764080.700341:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787764080.700343:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764080.700345:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764080.700346:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764080.700348:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764080.700350:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787764080.700352:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764080.700354:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764080.700356:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764080.700358:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764080.700360:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764080.700366:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764080.700367:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764080.700374:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787764080.700378:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764080.700380:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764080.700382:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764080.700384:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764080.700386:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764080.700389:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764080.700395:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787764080.700401:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764080.700410:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832689024/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764080.700421:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764080.700425:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764080.700430:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832689024/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764080.700451:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764080.700454:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764080.700457:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764080.700458:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764080.700460:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764080.700462:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764080.700465:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21795
00000400:00000200:0.0:1787764080.700469:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764080.700472:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764080.700473:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787764080.700474:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1787764080.700478:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764080.700479:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764080.700481:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764080.700483:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787764080.700485:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764080.700488:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764080.700490:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764080.700492:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00000040:2.0:1787764080.700492:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832689024/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787764080.700496:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764080.700500:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787764080.700505:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787764080.700510:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787764080.700511:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764080.700514:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764080.700515:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764080.700519:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1787764080.700519:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:0.0:1787764080.700524:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764080.700524:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764080.700530:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:2.0:1787764080.700530:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787764080.700535:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00100000:2.0:1787764080.700537:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832689024:192.168.203.110@tcp:250:kworker.0
00000100:00080000:0.0:1787764080.700541:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764080.700545:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764080.700548:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764080.700550:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764080.700551:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764080.700553:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1787764080.700554:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:2.0:1787764080.700559:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:0.0:1787764080.700560:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764080.700564:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764080.700565:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764080.700569:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764080.700572:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:2.0:1787764080.700573:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000008:00000001:0.0:1787764080.700578:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764080.700579:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000100:00000001:0.0:1787764080.700581:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764080.700587:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000400:00000010:2.0:1787764080.700589:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000020:00000040:0.0:1787764080.700590:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787764080.700594:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764080.700595:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764080.700597:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764080.700598:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:2.0:1787764080.700598:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832689024, portal 25
02000000:00000001:0.0:1787764080.700602:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764080.700602:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764080.700606:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
02000000:00000010:0.0:1787764080.700609:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000040:2.0:1787764080.700612:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832689024/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764105 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764080.700615:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764080.700623:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00100000:0.0:1787764080.700625:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832689152/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:2.0:1787764080.700630:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764080.700636:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000001:0.0:1787764080.700637:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:2.0:1787764080.700639:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832689024, offset 0
00000100:00000001:0.0:1787764080.700641:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764080.700646:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832689152/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764080.700649:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764080.700660:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764080.700681:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:0.0:1787764080.700682:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764080.700687:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787764080.700687:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:0.0:1787764080.700692:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:2.0:1787764080.700694:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:0.0:1787764080.700695:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764080.700699:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1787764080.700700:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1787764080.700702:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:2.0:1787764080.700705:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00080000:0.0:1787764080.700709:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21795
00000400:00000200:2.0:1787764080.700715:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58618:58618:256:4294967295] 192.168.203.110@tcp LPNI seq info [58618:58618:8:4294967295]
00000400:00000200:0.0:1787764080.700719:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764080.700724:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764080.700725:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764080.700728:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764080.700732:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764080.700737:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:2.0:1787764080.700757:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:0.0:1787764080.700760:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764080.700765:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764080.700769:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:2.0:1787764080.700770:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000020:00000040:0.0:1787764080.700771:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787764080.700776:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764080.700778:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000010:2.0:1787764080.700779:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000020:00000001:0.0:1787764080.700781:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787764080.700785:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787764080.700790:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:2.0:1787764080.700792:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787764080.700793:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764080.700801:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000200:2.0:1787764080.700801:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000100:00000001:0.0:1787764080.700804:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764080.700809:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000800:00000200:2.1:1787764080.700813:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00080000:0.0:1787764080.700816:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764080.700821:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787764080.700822:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
02000000:00000001:0.0:1787764080.700824:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764080.700828:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764080.700831:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:2.0:1787764080.700836:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787764080.700837:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764080.700841:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000800:00000010:2.0:1787764080.700844:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000008:00000001:0.0:1787764080.700845:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764080.700852:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764080.700857:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000010:0.0:1787764080.700858:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787764080.700865:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000400:00000200:2.0:1787764080.700869:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:0.0:1787764080.700872:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764080.700876:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764080.700878:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000100:00000001:0.0:1787764080.700880:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:2.0:1787764080.700884:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
02000000:00000001:0.0:1787764080.700885:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764080.700891:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764080.700894:0:257575:0:(events.c:34:request_out_callback()) Process entered
02000000:00000010:0.0:1787764080.700902:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000200:2.0:1787764080.700903:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876ed00 x1874599832689024/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764105 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764080.700912:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764080.700917:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764080.700930:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832689024/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764105 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787764080.700931:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832689280/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764080.700955:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764080.700960:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787764080.700965:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764080.700968:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764080.700972:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764080.700977:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764080.700981:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316468979
00000100:00000040:0.0:1787764080.700982:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832689280/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1787764080.700989:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7720.
00000100:00000001:0.0:1787764080.700995:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764080.701000:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764080.701000:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000040:0.0:1787764080.701008:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1787764080.701011:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316468979
00000100:00080000:0.0:1787764080.701012:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000010:2.0:1787764080.701018:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb76c0.
00000100:00080000:0.0:1787764080.701019:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787764080.701023:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787764080.701025:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00080000:0.0:1787764080.701026:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764080.701029:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764080.701031:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787764080.701033:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764080.701038:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21795
00000100:00000001:2.0:1787764080.701038:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000400:00000200:0.0:1787764080.701049:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00100000:2.0:1787764080.701050:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764080/real 1787764080]  req@ffff9751f876ed00 x1874599832689024/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787764080.701055:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764080.701061:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764080.701084:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000001:2.0:1787764080.701085:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:0.0:1787764080.701086:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000001:2.0:1787764080.701088:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:0.0:1787764080.701091:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000010:2.0:1787764080.701092:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000100:00000001:0.0:1787764080.701096:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787764080.701098:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000020:00000001:0.0:1787764080.701099:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000010:2.0:1787764080.701101:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000020:00000040:0.0:1787764080.701102:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787764080.701105:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1787764080.701106:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000040:0.0:1787764080.701108:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787764080.701112:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787764080.701114:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000200:2.0:1787764080.701116:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832689024/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764080.701118:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764080.701122:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764080.701127:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764080.701149:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764080.701158:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000200:2.0:1787764080.701165:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832689024/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787764080.701168:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764080.701176:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764080.701208:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764080.701212:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764080.701214:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764080.701217:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764080.701222:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:2.0:1787764080.701223:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000008:00000001:0.0:1787764080.701228:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764080.701229:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764080.701232:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764080.701234:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764080.701237:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:0.0:1787764080.701238:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787764080.701243:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00100000:2.0:1787764080.701244:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832689024/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787764080.701248:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764080.701251:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764080.701257:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1787764080.701260:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787764080.701261:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764080.701280:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764080.701282:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832689024/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1787764080.701285:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787764080.701291:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764080.701292:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764080.701297:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.701299:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764080.701300:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787764080.701303:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832689408/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764080.701303:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764080.701306:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764080.701311:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764080.701319:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:0.0:1787764080.701323:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764080.701323:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:0.0:1787764080.701330:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764080.701331:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832689024/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764080.701341:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832689408/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764080.701344:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832689024:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764080.701351:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.701355:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764080.701360:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832689024/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764080.701363:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764080.701370:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21805)
00000100:00000001:2.0:1787764080.701372:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764080.701377:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764080.701380:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764080.701385:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764080.701387:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764080.701390:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764080.701410:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764080.701413:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015c00.
02000000:00000001:2.0:1787764080.701418:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764080.701421:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764080.701423:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:2.0:1787764080.701428:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764080.701431:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.701433:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.701441:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764080.701442:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764080.701444:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764080.701446:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764080.701447:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764080.701451:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f0c0 x1874599832689152/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.701474:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764080.701475:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764080.701477:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764080.701479:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764080.701480:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764080.701482:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832689152:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764080.701485:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764080.701487:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764080.701488:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764080.701489:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764080.701492:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764080.701493:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764080.701496:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764080.701498:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764080.701501:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764080.701503:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832689152, portal 10
00000100:00000001:2.0:1787764080.701505:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764080.701506:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049530560 : -115096660021056 : ffff9751f876f0c0)
00000100:00000040:2.0:1787764080.701511:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f0c0 x1874599832689152/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764105 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.701516:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764080.701517:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764080.701519:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764080.701521:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832689152, offset 0
00000400:00000200:2.0:1787764080.701540:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764080.701544:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764080.701548:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764080.701550:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764080.701553:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764080.701555:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764080.701557:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764080.701560:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58619:58619:256:4294967295] 192.168.203.110@tcp LPNI seq info [58619:58619:8:4294967295]
00000400:00000200:2.0:1787764080.701563:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764080.701569:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764080.701573:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764080.701576:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764080.701579:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764080.701582:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787764080.701586:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764080.701588:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1787764080.701607:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764080.701609:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764080.701611:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764080.701613:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764080.701615:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764080.701617:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764080.701618:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764080.701621:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f0c0 x1874599832689152/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764105 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.701626:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764080.701628:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f0c0 x1874599832689152/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764105 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.701637:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764080.701638:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764080.701641:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764080.701644:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764080.701646:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764080.701647:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316467744
00000100:00000001:2.0:1787764080.701649:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316468979
00000100:00000001:2.0:1787764080.701651:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316467744
00000100:00000001:2.0:1787764080.701652:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316468979
00000100:00000001:2.0:1787764080.701654:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764080.701655:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764080.701657:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764080.701659:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764080/real 1787764080]  req@ffff9751f876f0c0 x1874599832689152/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764080.701664:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764080.701665:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764080.701666:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764080.701667:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764080.701685:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764080.701687:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764080.701690:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f0c0 x1874599832689152/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764080.701696:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f0c0 x1874599832689152/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.701700:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764080.701701:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.701702:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764080.701704:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764080.701724:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f0c0 x1874599832689152/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.701731:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764080.701734:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f0c0 x1874599832689152/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.701740:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764080.701742:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.701743:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764080.701744:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.701746:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764080.701757:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764080.701759:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764080.701762:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764080.701763:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764080.701767:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f0c0 x1874599832689152/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764080.701773:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832689152:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764080.701777:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764080.701780:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832689280/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.701786:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764080.701787:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764080.701788:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764080.701790:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764080.701792:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764080.701794:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832689280:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764080.701797:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764080.701799:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764080.701801:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764080.701803:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764080.701806:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764080.701807:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764080.701810:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764080.701812:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764080.701815:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764080.701818:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832689280, portal 10
00000100:00000001:2.0:1787764080.701820:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764080.701822:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00000040:2.0:1787764080.701830:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832689280/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764105 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.701840:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764080.701843:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764080.701846:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764080.701850:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832689280, offset 0
00000400:00000200:2.0:1787764080.701855:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764080.701862:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764080.701867:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764080.701871:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764080.701875:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764080.701878:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764080.701882:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764080.701886:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58620:58620:256:4294967295] 192.168.203.110@tcp LPNI seq info [58620:58620:8:4294967295]
00000400:00000200:2.0:1787764080.701892:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764080.701900:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764080.701904:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764080.701907:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764080.701911:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764080.701914:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787764080.701917:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764080.701920:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1787764080.701923:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764080.701926:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764080.701928:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764080.701931:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764080.701934:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764080.701936:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764080.701938:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764080.701941:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e580 x1874599832689280/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764105 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.701949:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764080.701951:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832689280/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764105 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.701957:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764080.701959:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764080.701961:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764080.701964:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764080.701967:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764080.701969:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316467732
00000100:00000001:2.0:1787764080.701971:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316468979
00000100:00000001:2.0:1787764080.701973:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316467732
00000100:00000001:2.0:1787764080.701975:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316468979
00000100:00000001:2.0:1787764080.701977:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764080.701978:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764080.701980:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764080.701983:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764080/real 1787764080]  req@ffff9751f876e580 x1874599832689280/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764080.701990:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764080.701991:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764080.701992:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764080.701994:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764080.701996:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764080.701998:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764080.702001:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832689280/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764080.702008:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832689280/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.702013:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764080.702014:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.702015:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764080.702017:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764080.702019:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832689280/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.702024:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764080.702027:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832689280/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.702033:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764080.702035:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.702036:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764080.702037:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.702039:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764080.702040:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764080.702042:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764080.702045:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764080.702047:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764080.702050:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832689280/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764080.702057:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832689280:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764080.702060:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764080.702078:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832689408/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.702082:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764080.702083:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764080.702085:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764080.702086:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764080.702087:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764080.702089:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832689408:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764080.702091:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764080.702092:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764080.702093:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764080.702094:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764080.702097:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764080.702098:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764080.702101:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764080.702102:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764080.702105:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764080.702108:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832689408, portal 4
00000100:00000001:2.0:1787764080.702110:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764080.702112:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000100:00000040:2.0:1787764080.702118:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832689408/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764105 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.702125:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764080.702127:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764080.702129:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764080.702132:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832689408, offset 0
00000400:00000200:2.0:1787764080.702136:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764080.702140:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764080.702144:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764080.702147:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764080.702150:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764080.702152:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764080.702155:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764080.702158:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58621:58621:256:4294967295] 192.168.203.110@tcp LPNI seq info [58621:58621:8:4294967295]
00000400:00000200:2.0:1787764080.702161:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764080.702181:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764080.702184:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764080.702187:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764080.702190:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764080.702192:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:2.1:1787764080.702195:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764080.702197:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:2.0:1787764080.702199:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764080.702201:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764080.702203:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764080.702205:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764080.702207:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764080.702208:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764080.702210:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764080.702212:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e1c0 x1874599832689408/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764105 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.702218:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764080.702220:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832689408/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764105 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.702225:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764080.702226:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764080.702228:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764080.702230:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764080.702232:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764080.702233:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316467750
00000100:00000001:2.0:1787764080.702235:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316468979
00000100:00000001:2.0:1787764080.702237:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316467750
00000100:00000001:2.0:1787764080.702238:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316468979
00000100:00000001:2.0:1787764080.702239:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764080.702241:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764080.702243:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764080.702245:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764080/real 1787764080]  req@ffff9751f876e1c0 x1874599832689408/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764080.702251:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764080.702252:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764080.702253:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764080.702254:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764080.702255:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764080.702256:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764080.702258:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832689408/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764080.702264:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832689408/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.702268:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764080.702269:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.702270:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764080.702271:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764080.702273:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832689408/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.702277:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764080.702279:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832689408/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.702283:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764080.702285:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.702286:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764080.702287:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.702288:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764080.702289:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764080.702291:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764080.702294:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764080.702295:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764080.702298:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832689408/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764080.702303:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832689408:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764080.702305:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.702306:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764080.702308:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f0c0 x1874599832689152/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.702312:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764080.702313:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764080.702314:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764080.702316:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764080.702317:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764080.702318:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764080.702319:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764080.702320:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016800.
02000000:00000001:2.0:1787764080.702323:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764080.702324:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764080.702325:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f0c0.
00000100:00000001:2.0:1787764080.702326:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764080.702327:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.702328:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764080.702330:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832689280/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.702334:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764080.702335:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764080.702336:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764080.702337:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764080.702338:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764080.702338:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764080.702339:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764080.702341:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:2.0:1787764080.702342:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764080.702343:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764080.702344:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:2.0:1787764080.702345:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764080.702346:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.702347:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764080.702349:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832689408/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764105 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764080.702353:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764080.702353:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764080.702354:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764080.702355:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764080.702356:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764080.702357:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764080.702358:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764080.702359:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014800.
02000000:00000001:2.0:1787764080.702360:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764080.702361:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764080.702361:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:2.0:1787764080.702363:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764080.702364:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.702365:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764080.702369:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764080.702370:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764080.702372:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764080.702373:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764081.340265:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764081.340271:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764081.340276:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764081.340280:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764081.340281:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764081.340293:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764081.340295:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764081.340300:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764081.340301:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764081.340308:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764081.340310:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764081.340315:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764081.340319:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764081.340320:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764081.340325:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764081.340327:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764081.340331:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764081.340338:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764081.340340:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764081.340344:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764081.340348:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764081.340417:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764081.340422:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764081.340425:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764081.724298:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764081.724308:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764081.724311:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764081.724318:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764081.724323:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764081.724327:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764081.724330:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764081.724333:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764081.724337:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764081.980248:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316468979, at 4316468992
00000100:00000010:3.0:1787764081.980273:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb76c0.
00000100:00000040:2.0:1787764081.980303:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316468979, at 4316468992
00000100:00000010:2.0:1787764081.980323:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7720.
00010000:00000001:3.0:1787764082.364266:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764082.364269:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764082.364274:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764082.364277:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764082.364280:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764082.364281:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764082.364285:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764082.364288:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764082.364291:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764082.364293:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764082.364295:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764082.364308:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764082.364312:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764082.364314:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764082.364317:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764082.364398:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764082.364402:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764082.364405:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764082.748261:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764082.748268:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764082.748271:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764082.748276:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764082.748281:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764082.748285:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764082.748287:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764082.748289:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764082.748291:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764083.388247:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764083.388254:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764083.388257:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764083.388257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764083.388265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764083.388266:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764083.388274:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764083.388277:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764083.388277:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764083.388284:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764083.388284:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764083.388288:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764083.388289:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764083.388293:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764083.388296:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764083.388347:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764083.388352:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764083.388357:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764083.772280:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764083.772290:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764083.772293:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764083.772300:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764083.772305:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764083.772316:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764083.772319:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764083.772323:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764083.772328:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764084.412248:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764084.412257:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764084.412261:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764084.412266:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764084.412267:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764084.412278:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764084.412279:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764084.412290:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764084.412292:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764084.412294:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764084.412301:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764084.412302:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764084.412307:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764084.412309:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764084.412311:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764084.412316:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764084.412317:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764084.412334:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764084.412337:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764084.412341:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764084.412343:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764084.412407:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764084.412410:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764084.412413:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764084.796269:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764084.796277:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764084.796280:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764084.796286:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764084.796290:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764084.796295:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764084.796297:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764084.796299:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764084.796302:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764085.436208:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764085.436211:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764085.436215:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764085.436217:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764085.436219:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764085.436221:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764085.436222:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764085.436223:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764085.436226:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764085.436227:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764085.436228:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764085.436230:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.436233:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764085.436236:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.436240:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764085.436292:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764085.436297:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764085.436300:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787764085.820245:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764085.820258:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787764085.820261:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764085.820265:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764085.820270:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764085.820279:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21800
00000100:00000001:1.0:1787764085.820283:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764085.820288:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000400:00000200:0.0:1787764085.820289:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764085.820291:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764085.820296:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000400:00000200:0.0:1787764085.820299:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1787764085.820300:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1787764085.820305:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787764085.820306:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764085.820311:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:0.0:1787764085.820313:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764085.820314:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764085.820319:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787764085.820320:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764085.820324:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764085.820328:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764085.820332:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764085.820335:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764085.820337:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764085.820342:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764085.820345:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764085.820348:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787764085.820352:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764085.820354:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764085.820356:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764085.820358:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764085.820362:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787764085.820365:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764085.820368:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764085.820371:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764085.820374:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764085.820377:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764085.820379:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764085.820382:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764085.820390:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787764085.820393:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764085.820397:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764085.820399:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764085.820402:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764085.820404:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764085.820417:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764085.820425:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787764085.820431:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764085.820440:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832689536/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764085.820451:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764085.820454:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764085.820457:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832689536/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764085.820472:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764085.820475:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764085.820477:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764085.820479:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764085.820481:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764085.820484:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764085.820494:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21800
00000100:00000001:2.0:1787764085.820496:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:0.0:1787764085.820501:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764085.820502:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000400:00000200:0.0:1787764085.820507:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787764085.820507:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00080000:0.0:1787764085.820513:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764085.820517:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832689536/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764085.820520:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764085.820525:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764085.820529:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764085.820531:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764085.820535:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764085.820535:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764085.820541:0:304425:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:2.0:1787764085.820541:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:0.0:1787764085.820543:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
02000000:00000001:2.0:1787764085.820545:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:0.0:1787764085.820549:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:2.0:1787764085.820549:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764085.820555:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832689536:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764085.820558:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764085.820562:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:0.0:1787764085.820563:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
02000000:00000001:2.0:1787764085.820564:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764085.820570:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:2.0:1787764085.820570:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000001:0.0:1787764085.820575:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000010:2.0:1787764085.820576:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764085.820580:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764085.820583:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764085.820588:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764085.820589:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1787764085.820592:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000100:00000040:0.0:1787764085.820595:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000010:2.0:1787764085.820600:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000001:0.0:1787764085.820601:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764085.820607:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00000200:2.0:1787764085.820610:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832689536, portal 25
00000100:00080000:0.0:1787764085.820611:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764085.820613:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787764085.820616:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764085.820617:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
02000000:00000001:0.0:1787764085.820621:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:2.0:1787764085.820626:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832689536/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764110 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787764085.820628:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764085.820646:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764085.820651:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764085.820654:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000008:00100000:0.0:1787764085.820657:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000040:2.0:1787764085.820657:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764085.820661:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000008:00000001:0.0:1787764085.820662:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764085.820665:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832689536, offset 0
00000100:00000001:0.0:1787764085.820666:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764085.820675:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000400:00000200:2.0:1787764085.820676:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000020:00000040:0.0:1787764085.820681:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787764085.820685:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764085.820686:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764085.820689:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764085.820691:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764085.820694:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764085.820696:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764085.820698:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
02000000:00000010:0.0:1787764085.820702:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000400:00000200:2.0:1787764085.820704:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1787764085.820709:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764085.820710:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764085.820715:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764085.820720:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00100000:0.0:1787764085.820722:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832689664/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787764085.820725:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58622:58622:256:4294967295] 192.168.203.110@tcp LPNI seq info [58622:58622:8:4294967295]
00000400:00000200:2.0:1787764085.820736:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764085.820743:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764085.820748:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764085.820749:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:0.0:1787764085.820757:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832689664/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1787764085.820757:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764085.820764:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000100:00000001:0.0:1787764085.820774:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787764085.820774:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764085.820779:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000100:00080000:0.0:1787764085.820780:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764085.820786:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000800:00000200:2.1:1787764085.820787:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787764085.820790:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:2.1:1787764085.820792:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000100:00080000:0.0:1787764085.820794:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764085.820799:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000800:00000100:2.0:1787764085.820799:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764085.820803:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000100:00080000:0.0:1787764085.820804:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21800
00000400:00000200:2.0:1787764085.820808:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764085.820814:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764085.820818:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000200:0.0:1787764085.820819:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764085.820821:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000400:00000200:0.0:1787764085.820826:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787764085.820827:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:0.0:1787764085.820831:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000200:2.0:1787764085.820836:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876c000 x1874599832689536/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764110 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764085.820839:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764085.820846:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764085.820851:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764085.820854:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1787764085.820855:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764085.820861:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000040:2.0:1787764085.820861:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832689536/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764110 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787764085.820864:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787764085.820868:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764085.820871:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000100:00000001:2.0:1787764085.820873:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764085.820876:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000001:2.0:1787764085.820876:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:0.0:1787764085.820881:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000001:2.0:1787764085.820883:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764085.820887:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764085.820891:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764085.820892:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764085.820895:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316474099
00000100:00000040:0.0:1787764085.820898:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764085.820902:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000010:2.0:1787764085.820902:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7720.
00000020:00000040:0.0:1787764085.820908:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00000001:2.0:1787764085.820912:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00080000:0.0:1787764085.820913:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764085.820918:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764085.820921:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787764085.820921:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316474099
00000100:00000010:2.0:1787764085.820926:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7840.
02000000:00000001:0.0:1787764085.820927:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764085.820932:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764085.820932:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787764085.820938:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764085.820938:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000008:00100000:0.0:1787764085.820942:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1787764085.820942:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764085.820945:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000008:00000001:0.0:1787764085.820949:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764085.820950:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764085/real 1787764085]  req@ffff9751f876c000 x1874599832689536/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764085.820954:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764085.820961:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
00000400:00000001:2.0:1787764085.820962:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764085.820965:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000020:00000040:0.0:1787764085.820968:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000400:00000010:2.0:1787764085.820968:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764085.820972:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
02000000:00000001:0.0:1787764085.820975:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:2.0:1787764085.820975:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
02000000:00000001:0.0:1787764085.820979:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764085.820980:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764085.820985:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832689536/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764085.820986:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764085.820990:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764085.820994:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764085.820996:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832689536/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1787764085.821002:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:2.0:1787764085.821003:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764085.821005:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.821008:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:0.0:1787764085.821009:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764085.821010:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764085.821017:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832689536/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787764085.821022:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832689792/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764085.821024:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764085.821032:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832689536/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764085.821042:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764085.821044:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787764085.821048:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764085.821048:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.821054:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764085.821055:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.821058:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000040:0.0:1787764085.821060:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832689792/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821060:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764085.821066:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764085.821074:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764085.821078:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:0.0:1787764085.821082:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764085.821084:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832689536/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764085.821088:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1787764085.821093:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832689536:192.168.203.110@tcp:250:kworker.0
00000100:00000040:0.0:1787764085.821098:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:2.0:1787764085.821099:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.821104:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00080000:0.0:1787764085.821105:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1787764085.821106:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832689536/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787764085.821115:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787764085.821118:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
00000100:00000001:0.0:1787764085.821120:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
02000000:00000001:2.0:1787764085.821121:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00080000:0.0:1787764085.821125:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
02000000:00000010:2.0:1787764085.821126:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764085.821130:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000001:0.0:1787764085.821132:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000020:00000001:2.0:1787764085.821149:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764085.821151:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764085.821152:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764085.821154:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:2.0:1787764085.821158:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764085.821159:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764085.821161:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:2.0:1787764085.821163:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764085.821166:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787764085.821169:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21800
00000100:00000001:2.0:1787764085.821170:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1787764085.821178:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764085.821178:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764085.821182:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764085.821184:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:0.0:1787764085.821186:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787764085.821187:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764085.821189:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00080000:0.0:1787764085.821193:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764085.821195:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876de00 x1874599832689664/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764085.821202:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764085.821204:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764085.821206:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764085.821208:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1787764085.821209:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764085.821213:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1787764085.821216:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764085.821216:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764085.821222:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00100000:2.0:1787764085.821223:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832689664:192.168.203.110@tcp:38:kworker.0
00000020:00000001:0.0:1787764085.821230:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1787764085.821230:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:0.0:1787764085.821234:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
02000000:00000001:2.0:1787764085.821235:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764085.821237:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764085.821239:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:2.0:1787764085.821240:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764085.821244:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
00000020:00000040:0.0:1787764085.821245:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
02000000:00000001:2.0:1787764085.821249:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764085.821251:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787764085.821257:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000200:2.0:1787764085.821257:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764085.821263:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000100:00000040:0.0:1787764085.821265:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764085.821270:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1787764085.821271:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000040:0.0:1787764085.821274:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000200:2.0:1787764085.821277:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832689664, portal 10
00000100:00000001:0.0:1787764085.821278:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764085.821285:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:2.0:1787764085.821285:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764085.821290:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049525760 : -115096660025856 : ffff9751f876de00)
00000100:00080000:0.0:1787764085.821291:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764085.821297:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764085.821300:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764085.821305:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1787764085.821305:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876de00 x1874599832689664/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764110 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787764085.821311:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764085.821316:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764085.821320:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000008:00100000:0.0:1787764085.821321:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000040:2.0:1787764085.821325:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000008:00000001:0.0:1787764085.821327:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764085.821330:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764085.821330:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764085.821334:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832689664, offset 0
00000100:00000010:0.0:1787764085.821338:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000400:00000200:2.0:1787764085.821342:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000020:00000040:0.0:1787764085.821344:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000400:00000200:2.0:1787764085.821349:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787764085.821350:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764085.821353:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764085.821354:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:0.0:1787764085.821357:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:2.0:1787764085.821359:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:0.0:1787764085.821361:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1787764085.821363:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
02000000:00000001:0.0:1787764085.821366:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764085.821367:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764085.821371:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764085.821374:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58623:58623:256:4294967295] 192.168.203.110@tcp LPNI seq info [58623:58623:8:4294967295]
02000000:00000010:0.0:1787764085.821375:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000400:00000200:2.0:1787764085.821382:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764085.821383:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764085.821391:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00100000:0.0:1787764085.821396:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832689920/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.0:1787764085.821400:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764085.821406:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764085.821414:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787764085.821415:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:2.0:1787764085.821416:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:0.0:1787764085.821420:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.1:1787764085.821424:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764085.821428:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000100:00000040:0.0:1787764085.821433:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832689920/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:2.0:1787764085.821433:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764085.821436:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764085.821440:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764085.821448:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764085.821454:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764085.821457:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:0.0:1787764085.821458:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764085.821459:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764085.821465:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876de00 x1874599832689664/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764110 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764085.821466:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21810)
00000100:00000001:2.0:1787764085.821475:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764085.821477:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876de00 x1874599832689664/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764110 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821483:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764085.821484:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764085.821487:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764085.821490:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764085.821492:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764085.821494:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316472864
00000100:00000001:2.0:1787764085.821496:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316474099
00000100:00000001:2.0:1787764085.821498:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316472864
00000100:00000001:2.0:1787764085.821499:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316474099
00000100:00000001:2.0:1787764085.821501:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764085.821502:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764085.821503:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764085.821506:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764085/real 1787764085]  req@ffff9751f876de00 x1874599832689664/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764085.821511:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764085.821512:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764085.821514:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764085.821515:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764085.821516:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764085.821517:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764085.821519:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876de00 x1874599832689664/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764085.821524:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876de00 x1874599832689664/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821529:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764085.821530:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.821531:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764085.821532:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764085.821534:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876de00 x1874599832689664/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821538:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764085.821540:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876de00 x1874599832689664/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821544:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764085.821546:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.821547:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764085.821548:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.821549:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764085.821558:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764085.821560:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764085.821562:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764085.821563:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764085.821566:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876de00 x1874599832689664/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764085.821571:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832689664:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764085.821574:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764085.821576:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876da40 x1874599832689792/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821580:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764085.821581:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764085.821582:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764085.821583:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764085.821584:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764085.821586:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832689792:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764085.821588:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764085.821589:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764085.821590:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764085.821591:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764085.821594:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764085.821595:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764085.821597:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764085.821598:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764085.821600:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764085.821602:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832689792, portal 10
00000100:00000001:2.0:1787764085.821603:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764085.821604:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049524800 : -115096660026816 : ffff9751f876da40)
00000100:00000040:2.0:1787764085.821608:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876da40 x1874599832689792/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764110 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821612:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764085.821613:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764085.821615:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764085.821616:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832689792, offset 0
00000400:00000200:2.0:1787764085.821619:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764085.821621:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764085.821623:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764085.821625:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764085.821627:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764085.821628:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764085.821630:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764085.821632:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58624:58624:256:4294967295] 192.168.203.110@tcp LPNI seq info [58624:58624:8:4294967295]
00000400:00000200:2.0:1787764085.821649:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764085.821654:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764085.821657:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764085.821660:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764085.821662:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764085.821664:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787764085.821667:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764085.821668:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1787764085.821671:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764085.821672:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764085.821674:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764085.821676:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764085.821678:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764085.821680:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764085.821681:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764085.821684:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876da40 x1874599832689792/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764110 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821689:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764085.821691:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876da40 x1874599832689792/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764110 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821695:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764085.821696:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764085.821698:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764085.821700:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764085.821702:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764085.821704:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316472852
00000100:00000001:2.0:1787764085.821705:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316474099
00000100:00000001:2.0:1787764085.821707:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316472852
00000100:00000001:2.0:1787764085.821708:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316474099
00000100:00000001:2.0:1787764085.821709:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764085.821711:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764085.821712:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764085.821714:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764085/real 1787764085]  req@ffff9751f876da40 x1874599832689792/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764085.821719:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764085.821720:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764085.821721:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764085.821723:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764085.821724:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764085.821725:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764085.821727:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876da40 x1874599832689792/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764085.821733:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876da40 x1874599832689792/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821736:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764085.821737:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.821738:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764085.821739:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764085.821741:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876da40 x1874599832689792/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821745:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764085.821747:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876da40 x1874599832689792/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821751:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764085.821753:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.821754:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764085.821755:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.821757:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764085.821758:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764085.821760:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764085.821762:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764085.821763:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764085.821765:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876da40 x1874599832689792/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764085.821770:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832689792:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764085.821772:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.821773:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764085.821775:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876de00 x1874599832689664/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821779:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764085.821780:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764085.821781:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764085.821782:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764085.821783:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764085.821784:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764085.821785:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764085.821787:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016c00.
02000000:00000001:2.0:1787764085.821789:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764085.821790:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764085.821791:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876de00.
00000100:00000001:2.0:1787764085.821794:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764085.821795:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.821796:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764085.821798:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876da40 x1874599832689792/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821806:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764085.821807:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764085.821808:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764085.821810:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764085.821810:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764085.821811:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764085.821812:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764085.821813:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017800.
02000000:00000001:2.0:1787764085.821815:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764085.821816:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764085.821817:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876da40.
00000100:00000001:2.0:1787764085.821818:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764085.821819:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.821820:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.821822:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764085.821823:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764085.821824:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764085.821825:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764085.821825:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764085.821828:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cb40 x1874599832689920/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821832:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764085.821833:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764085.821834:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764085.821835:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764085.821836:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764085.821838:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832689920:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764085.821840:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764085.821841:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764085.821843:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764085.821844:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764085.821846:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764085.821847:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764085.821849:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764085.821851:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764085.821853:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764085.821855:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832689920, portal 4
00000100:00000001:2.0:1787764085.821856:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764085.821857:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049520960 : -115096660030656 : ffff9751f876cb40)
00000100:00000040:2.0:1787764085.821861:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cb40 x1874599832689920/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764110 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821865:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764085.821867:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764085.821868:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764085.821870:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832689920, offset 0
00000400:00000200:2.0:1787764085.821872:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764085.821874:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764085.821876:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764085.821878:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764085.821880:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764085.821881:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764085.821883:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764085.821885:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58625:58625:256:4294967295] 192.168.203.110@tcp LPNI seq info [58625:58625:8:4294967295]
00000400:00000200:2.0:1787764085.821888:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764085.821892:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764085.821894:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764085.821897:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764085.821899:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764085.821901:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:2.1:1787764085.821903:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764085.821905:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:2.0:1787764085.821908:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764085.821909:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764085.821911:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764085.821913:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764085.821915:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764085.821916:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764085.821917:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764085.821920:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cb40 x1874599832689920/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764110 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821925:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764085.821926:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cb40 x1874599832689920/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764110 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821930:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764085.821931:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764085.821933:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764085.821935:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764085.821937:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764085.821938:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316472869
00000100:00000001:2.0:1787764085.821940:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316474099
00000100:00000001:2.0:1787764085.821941:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316472869
00000100:00000001:2.0:1787764085.821943:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316474099
00000100:00000001:2.0:1787764085.821944:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764085.821945:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764085.821946:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764085.821948:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764085/real 1787764085]  req@ffff9751f876cb40 x1874599832689920/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764085.821953:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764085.821954:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764085.821955:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764085.821956:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764085.821957:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764085.821958:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764085.821960:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cb40 x1874599832689920/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764085.821965:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cb40 x1874599832689920/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821970:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764085.821971:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.821972:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764085.821973:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764085.821975:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cb40 x1874599832689920/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821979:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764085.821981:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cb40 x1874599832689920/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.821985:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764085.821986:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.821987:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764085.821988:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.821989:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764085.821990:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764085.821992:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764085.821993:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764085.821995:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764085.821997:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cb40 x1874599832689920/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764085.822001:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832689920:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764085.822003:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.822004:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764085.822006:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cb40 x1874599832689920/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764110 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764085.822010:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764085.822011:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764085.822013:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764085.822014:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764085.822015:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764085.822016:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764085.822017:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764085.822018:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:2.0:1787764085.822019:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764085.822020:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764085.822021:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cb40.
00000100:00000001:2.0:1787764085.822022:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764085.822023:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.822024:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764085.822026:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764085.822027:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764085.822028:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764085.822029:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764086.460239:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764086.460242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764086.460246:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764086.460247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764086.460249:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764086.460253:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764086.460254:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764086.460257:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764086.460259:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764086.460262:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764086.460264:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764086.460265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764086.460271:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764086.460273:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764086.460276:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764086.460327:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764086.460331:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764086.460335:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764086.844234:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764086.844241:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764086.844243:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764086.844251:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764086.844255:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764086.844258:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764086.844260:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764086.844262:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764086.844264:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764087.100216:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316474099, at 4316474112
00000100:00000010:3.0:1787764087.100231:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7840.
00000100:00000040:2.0:1787764087.100254:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316474099, at 4316474112
00000100:00000010:2.0:1787764087.100262:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7720.
00010000:00000001:3.0:1787764087.484195:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764087.484202:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764087.484207:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764087.484210:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764087.484212:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764087.484213:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764087.484215:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764087.484220:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764087.484221:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764087.484222:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764087.484226:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764087.484226:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764087.484229:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764087.484233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764087.484234:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764087.484235:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764087.484241:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764087.484244:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764087.484247:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764087.484250:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764087.484253:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764087.484291:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764087.484294:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764087.484297:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764087.868225:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764087.868234:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764087.868236:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764087.868241:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764087.868245:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764087.868248:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764087.868251:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764087.868254:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764087.868256:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764088.508241:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764088.508256:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764088.508262:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764088.508263:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764088.508269:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764088.508271:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764088.508278:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764088.508279:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764088.508282:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764088.508287:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764088.508290:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764088.508293:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764088.508298:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764088.508302:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764088.508306:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764088.508372:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764088.508377:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764088.508382:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764088.892263:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764088.892271:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764088.892273:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764088.892279:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764088.892283:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764088.892287:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764088.892290:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764088.892292:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764088.892294:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764089.532226:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764089.532234:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764089.532237:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764089.532243:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764089.532244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764089.532248:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764089.532250:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764089.532253:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764089.532258:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764089.532259:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764089.532264:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764089.532265:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764089.532271:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764089.532272:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764089.532275:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764089.532340:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764089.532345:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764089.532349:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764089.916245:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764089.916251:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764089.916253:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764089.916257:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764089.916261:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764089.916263:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764089.916266:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764089.916268:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764089.916270:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764090.556232:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764090.556232:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764090.556238:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764090.556240:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764090.556245:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764090.556248:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764090.556250:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764090.556252:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764090.556257:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764090.556259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764090.556262:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764090.556264:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764090.556275:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764090.556279:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764090.556283:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764090.556330:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764090.556334:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764090.556337:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764090.940251:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1787764090.940258:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764090.940259:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764090.940261:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764090.940266:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:3.0:1787764090.940266:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764090.940269:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787764090.940271:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:3.0:1787764090.940272:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764090.940274:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764090.940276:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787764090.940277:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764090.940280:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764090.940283:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764090.940284:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21805
00000400:00000200:3.0:1787764090.940291:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764090.940298:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764090.940302:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764090.940306:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764090.940309:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764090.940312:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764090.940315:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764090.940319:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764090.940321:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787764090.940324:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764090.940328:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787764090.940330:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787764090.940333:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787764090.940336:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764090.940338:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764090.940347:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764090.940349:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764090.940352:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787764090.940355:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764090.940357:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764090.940360:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764090.940363:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764090.940365:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764090.940367:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764090.940370:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764090.940378:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bde00.
00000020:00000040:3.0:1787764090.940382:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787764090.940387:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764090.940389:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764090.940393:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764090.940396:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764090.940399:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764090.940407:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611c00.
00000100:00000001:3.0:1787764090.940414:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764090.940424:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bde00 x1874599832690048/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764090.940435:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764090.940440:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764090.940446:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000941163c2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bde00 x1874599832690048/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764090.940470:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764090.940496:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764090.940501:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764090.940504:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764090.940511:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bde00 x1874599832690048/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764090.940521:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764090.940524:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764090.940528:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764090.940531:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764090.940534:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764090.940540:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832690048:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787764090.940545:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764090.940550:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:3.0:1787764090.940551:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:1.0:1787764090.940554:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764090.940557:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
02000000:00000001:1.0:1787764090.940560:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1787764090.940560:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
02000000:00000010:1.0:1787764090.940567:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c8000.
02000000:00000001:1.0:1787764090.940569:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764090.940575:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764090.940579:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210630.
00000400:00000010:1.0:1787764090.940584:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933d48.
00000100:00000200:1.0:1787764090.940590:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832690048, portal 25
00000100:00000001:1.0:1787764090.940592:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764090.940593:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049853440 : -115096659698176 : ffff9751f87bde00)
00000100:00000040:1.0:1787764090.940609:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bde00 x1874599832690048/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764115 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764090.940616:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764090.940618:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764090.940621:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79332a8.
00000100:00000200:1.0:1787764090.940624:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832690048, offset 0
00000400:00000200:1.0:1787764090.940628:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764090.940637:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764090.940645:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764090.940651:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764090.940656:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764090.940660:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764090.940664:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764090.940669:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58626:58626:256:4294967295] 192.168.203.110@tcp LPNI seq info [58626:58626:8:4294967295]
00000400:00000200:1.0:1787764090.940675:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:3.0:1787764090.940686:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1787764090.940689:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:3.0:1787764090.940691:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764090.940697:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21805
00000800:00000200:1.0:1787764090.940700:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764090.940704:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000010:1.0:1787764090.940706:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1a00.
00000400:00000200:3.0:1787764090.940710:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764090.940714:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000200:1.0:1787764090.940718:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:3.0:1787764090.940719:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764090.940724:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.1:1787764090.940729:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:3.0:1787764090.940732:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764090.940742:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764090.940747:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764090.940749:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316479219
00000100:00000010:1.0:1787764090.940754:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c43c0.
00000100:00000001:1.0:1787764090.940760:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787764090.940767:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316479219
00000100:00000010:1.0:1787764090.940770:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4ae0.
00000100:00000001:1.0:1787764090.940773:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787764090.940776:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764090.940778:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764090.940781:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764090.940782:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764090.940787:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787764090.940791:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000020:00000001:3.0:1787764090.940793:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:1.0:1787764090.940794:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000020:00000040:3.0:1787764090.940797:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00000001:1.0:1787764090.940798:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764090.940800:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1787764090.940802:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:1.0:1787764090.940803:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764090.940807:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1787764090.940808:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:3.0:1787764090.940813:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:3.0:1787764090.940816:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:3.0:1787764090.940821:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764090.940824:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764090.940827:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764090.940829:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764090.940832:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:3.0:1787764090.940835:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764090.940838:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764090.940839:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764090.940845:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764090.940848:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764090.940850:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787764090.940855:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787764090.940858:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764090.940860:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764090.940863:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87be1c0.
00000020:00000040:3.0:1787764090.940865:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787764090.940867:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764090.940868:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764090.940870:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764090.940871:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764090.940873:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764090.940877:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610c00.
00000100:00000001:3.0:1787764090.940880:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764090.940885:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87be1c0 x1874599832690176/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764090.940891:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764090.940892:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764090.940895:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e2011c6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87be1c0 x1874599832690176/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764090.940912:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764090.940915:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764090.940917:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1787764090.940918:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764090.940919:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764090.940921:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764090.940923:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21805
00000400:00000200:3.0:1787764090.940926:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764090.940929:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764090.940931:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764090.940934:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764090.940936:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787764090.940938:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1787764090.940939:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764090.940942:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764090.940943:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787764090.940947:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1787764090.940947:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764090.940950:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:3.0:1787764090.940952:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764090.940954:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000100:00000040:1.0:1787764090.940955:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87be1c0 x1874599832690176/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787764090.940956:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:3.0:1787764090.940961:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:3.0:1787764090.940970:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764090.940971:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764090.940974:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764090.940977:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1787764090.940978:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764090.940981:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1787764090.940984:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:1.0:1787764090.940985:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764090.940992:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832690176:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764090.940992:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787764090.940999:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:3.0:1787764090.941000:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
02000000:00000001:1.0:1787764090.941001:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764090.941005:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764090.941008:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:3.0:1787764090.941009:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000010:1.0:1787764090.941013:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9000.
00000100:00000001:3.0:1787764090.941014:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764090.941018:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764090.941019:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764090.941026:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1787764090.941028:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:3.0:1787764090.941030:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787764090.941034:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000100:00000001:3.0:1787764090.941035:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787764090.941040:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000010:1.0:1787764090.941042:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000008:00000001:3.0:1787764090.941044:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764090.941047:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787764090.941049:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832690176, portal 10
00000100:00000010:3.0:1787764090.941052:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc000.
00000100:00000001:1.0:1787764090.941054:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000020:00000040:3.0:1787764090.941055:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000001:1.0:1787764090.941058:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049854400 : -115096659697216 : ffff9751f87be1c0)
02000000:00000001:3.0:1787764090.941059:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764090.941061:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764090.941065:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764090.941066:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764090.941068:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764090.941073:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87be1c0 x1874599832690176/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764115 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1787764090.941074:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610800.
00000100:00000001:3.0:1787764090.941081:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764090.941087:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00100000:3.0:1787764090.941093:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc000 x1874599832690304/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1787764090.941094:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764090.941100:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79333b8.
00000100:00000200:1.0:1787764090.941106:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832690176, offset 0
00000400:00000200:1.0:1787764090.941112:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787764090.941114:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1787764090.941118:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764090.941120:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764090.941126:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000040:3.0:1787764090.941130:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3fc1c6c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc000 x1874599832690304/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787764090.941158:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1787764090.941166:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764090.941167:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000040:3.0:1787764090.941172:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787764090.941173:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000040:3.0:1787764090.941181:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:1.0:1787764090.941184:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00080000:3.0:1787764090.941186:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787764090.941194:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58627:58627:255:4294967295] 192.168.203.110@tcp LPNI seq info [58627:58627:7:4294967295]
00000100:00080000:3.0:1787764090.941195:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787764090.941198:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764090.941203:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1787764090.941205:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764090.941208:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764090.941216:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21805
00000400:00000200:1.0:1787764090.941221:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764090.941230:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764090.941232:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000010:1.0:1787764090.941238:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1200.
00000400:00000200:3.0:1787764090.941240:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764090.941245:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000200:1.0:1787764090.941248:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:3.0:1787764090.941253:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764090.941258:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.1:1787764090.941259:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787764090.941266:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764090.941266:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764090.941272:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:1.0:1787764090.941275:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000020:00000001:3.0:1787764090.941277:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764090.941281:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000400:00000200:1.0:1787764090.941282:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:3.0:1787764090.941284:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:1.0:1787764090.941286:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316477984
00000020:00000040:3.0:1787764090.941289:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000001:1.0:1787764090.941291:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316479219
00000100:00000001:1.0:1787764090.941296:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316477984
00000020:00000001:3.0:1787764090.941296:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:3.0:1787764090.941301:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:1.0:1787764090.941302:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316479219
00000100:00000001:1.0:1787764090.941308:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1787764090.941310:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764090.941312:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764090.941316:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787764090.941317:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764090.941322:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764090.941322:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764090.941329:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787764090.941330:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000020:00000040:3.0:1787764090.941333:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:1.0:1787764090.941334:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1787764090.941339:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:3.0:1787764090.941340:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764090.941342:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764090.941345:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764090.941346:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
02000000:00000001:3.0:1787764090.941350:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:1.0:1787764090.941356:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc000 x1874599832690304/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787764090.941356:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764090.941362:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764090.941366:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787764090.941371:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1787764090.941372:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00000001:3.0:1787764090.941376:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764090.941377:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764090.941382:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764090.941384:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764090.941387:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000010:3.0:1787764090.941389:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bed00.
02000000:00000001:1.0:1787764090.941391:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1787764090.941394:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00100000:1.0:1787764090.941398:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832690304:192.168.203.110@tcp:38:kworker.0
02000000:00000001:3.0:1787764090.941400:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764090.941404:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764090.941406:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:3.0:1787764090.941407:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764090.941410:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764090.941411:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764090.941414:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764090.941416:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764090.941418:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764090.941424:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612000.
02000000:00000010:1.0:1787764090.941425:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764090.941430:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764090.941430:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764090.941439:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00100000:3.0:1787764090.941439:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bed00 x1874599832690432/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1787764090.941445:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210d10.
00000400:00000010:1.0:1787764090.941454:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000001:3.0:1787764090.941455:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764090.941460:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787764090.941461:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832690304, portal 10
00000100:00000001:1.0:1787764090.941465:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000040:3.0:1787764090.941467:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000079751c9a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bed00 x1874599832690432/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764090.941468:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049845760 : -115096659705856 : ffff9751f87bc000)
00000100:00000040:1.0:1787764090.941482:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc000 x1874599832690304/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764115 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764090.941482:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764090.941491:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21815)
00000100:00000001:1.0:1787764090.941494:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764090.941499:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764090.941503:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933908.
00000100:00000200:1.0:1787764090.941506:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832690304, offset 0
00000400:00000200:1.0:1787764090.941513:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764090.941518:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764090.941523:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764090.941526:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764090.941529:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764090.941531:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764090.941534:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764090.941537:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58628:58628:254:4294967295] 192.168.203.110@tcp LPNI seq info [58628:58628:6:4294967295]
00000400:00000200:1.0:1787764090.941542:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764090.941549:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764090.941553:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764090.941557:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1000.
00000800:00000200:1.0:1787764090.941561:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764090.941566:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787764090.941569:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764090.941572:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787764090.941575:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764090.941577:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316477972
00000100:00000001:1.0:1787764090.941580:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316479219
00000100:00000001:1.0:1787764090.941582:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316477972
00000100:00000001:1.0:1787764090.941585:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316479219
00000100:00000001:1.0:1787764090.941588:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764090.941590:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764090.941592:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764090.941594:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764090.941616:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764090.941618:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1787764090.941621:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764090.941624:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764090.941625:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764090.941631:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bed00 x1874599832690432/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764090.941638:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764090.941639:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764090.941642:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764090.941644:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764090.941646:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764090.941649:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832690432:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764090.941653:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764090.941655:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764090.941656:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764090.941658:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764090.941664:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca000.
02000000:00000001:1.0:1787764090.941666:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764090.941670:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764090.941672:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210000.
00000400:00000010:1.0:1787764090.941676:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933550.
00000100:00000200:1.0:1787764090.941679:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832690432, portal 4
00000100:00000001:1.0:1787764090.941681:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764090.941682:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049857280 : -115096659694336 : ffff9751f87bed00)
00000100:00000040:1.0:1787764090.941688:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bed00 x1874599832690432/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764115 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764090.941695:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764090.941698:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764090.941701:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933110.
00000100:00000200:1.0:1787764090.941703:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832690432, offset 0
00000400:00000200:1.0:1787764090.941708:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764090.941712:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764090.941715:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764090.941718:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764090.941721:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764090.941724:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764090.941726:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764090.941728:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58629:58629:253:4294967295] 192.168.203.110@tcp LPNI seq info [58629:58629:5:4294967295]
00000400:00000200:1.0:1787764090.941733:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764090.941738:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764090.941742:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764090.941746:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1e00.
00000800:00000200:1.0:1787764090.941749:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764090.941753:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787764090.941755:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764090.941759:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787764090.941762:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764090.941764:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316477989
00000100:00000001:1.0:1787764090.941766:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316479219
00000100:00000001:1.0:1787764090.941768:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316477989
00000100:00000001:1.0:1787764090.941770:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316479219
00000100:00000001:1.0:1787764090.941772:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764090.941774:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764090.941776:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764090.941778:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764090.941781:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764090.941782:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:1.0:1787764090.941784:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764090.941785:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764090.941787:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764090.941789:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764091.580272:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764091.580279:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764091.580281:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764091.580286:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:3.0:1787764091.580289:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764091.580293:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764091.580295:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764091.580297:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764091.580299:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764091.580318:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764091.580320:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764091.580324:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764091.580325:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764091.580328:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764091.580331:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764091.580333:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764091.580336:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764091.580339:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764091.580341:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764091.580343:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764091.580344:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764091.580406:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764091.580410:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764091.580413:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764091.964237:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764091.964244:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764091.964246:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764091.964251:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764091.964255:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764091.964258:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764091.964260:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764091.964262:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764091.964265:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764092.220238:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316479219, at 4316479232
00000100:00000010:3.0:1787764092.220261:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4ae0.
00000100:00000040:2.0:1787764092.220267:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316479219, at 4316479232
00000100:00000010:2.0:1787764092.220276:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c43c0.
00000100:00000001:2.0:1787764092.604283:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764092.604283:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764092.604290:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764092.604291:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764092.604294:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764092.604299:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764092.604299:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764092.604304:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764092.604305:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764092.604309:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764092.604310:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764092.604312:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764092.604314:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764092.604315:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764092.604320:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764092.604379:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764092.604383:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764092.604387:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764092.988230:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764092.988239:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764092.988242:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764092.988248:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764092.988252:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764092.988256:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764092.988259:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764092.988262:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764092.988266:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764093.628252:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764093.628261:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764093.628263:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764093.628265:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764093.628269:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764093.628271:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764093.628274:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764093.628279:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764093.628280:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764093.628283:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764093.628285:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764093.628290:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764093.628291:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764093.628295:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764093.628296:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764093.628348:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764093.628352:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764093.628355:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764094.012254:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764094.012261:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764094.012264:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764094.012268:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764094.012272:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764094.012275:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764094.012277:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764094.012279:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764094.012281:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:2.0:1787764094.012409:0:257563:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.110:988
00000400:00000100:2.0:1787764094.012449:0:257563:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.110@tcp at host 192.168.203.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1787764094.012460:0:257563:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764094.012473:0:257563:0:(router.c:1791:lnet_notify()) 192.168.203.10@tcp notifying 192.168.203.110@tcp: down
00000400:00000200:2.0:1787764094.012478:0:257563:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764094.012483:0:257563:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1787764094.012488:0:257563:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:2.0:1787764094.012492:0:257563:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1a00.
00000400:00000200:2.0:1787764094.012497:0:257563:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787764094.012503:0:257563:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787764094.012508:0:257563:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79332a8
00000400:00000010:2.0:1787764094.012511:0:257563:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79332a8.
00000100:00000001:2.0:1787764094.012516:0:257563:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764094.012525:0:257563:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f87bde00 x1874599832690048/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764115 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764094.012548:0:257563:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764094.012553:0:257563:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bde00 x1874599832690048/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764115 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764094.012562:0:257563:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764094.012564:0:257563:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787764094.012574:0:257563:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:2.0:1787764094.012589:0:257563:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1200.
00000400:00000200:2.0:1787764094.012594:0:257563:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787764094.012599:0:257563:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787764094.012603:0:257563:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79333b8
00000400:00000010:2.0:1787764094.012606:0:257563:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79333b8.
00000100:00000001:2.0:1787764094.012610:0:257563:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:1.0:1787764094.012614:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000200:2.0:1787764094.012616:0:257563:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f87be1c0 x1874599832690176/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764115 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764094.012621:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764094.012627:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1787764094.012630:0:257563:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764094.012639:0:257563:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87be1c0 x1874599832690176/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764115 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764094.012641:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764090/real 1787764094]  req@ffff9751f87bde00 x1874599832690048/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764094.012660:0:257563:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764094.012665:0:257563:0:(events.c:68:request_out_callback()) Process leaving
00000400:00000001:1.0:1787764094.012669:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764094.012673:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000800:00000100:2.0:1787764094.012674:0:257563:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000400:00000010:1.0:1787764094.012679:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210630.
00000800:00000010:2.0:1787764094.012679:0:257563:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1000.
00000400:00000200:1.0:1787764094.012688:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933d48
00000400:00000200:2.0:1787764094.012688:0:257563:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000010:1.0:1787764094.012694:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933d48.
00000400:00000200:2.0:1787764094.012694:0:257563:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000100:00000001:1.0:1787764094.012702:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000400:00000200:2.0:1787764094.012703:0:257563:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933908
00000400:00000010:2.0:1787764094.012707:0:257563:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933908.
00000100:00000200:1.0:1787764094.012711:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bde00 x1874599832690048/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764094.012713:0:257563:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764094.012721:0:257563:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f87bc000 x1874599832690304/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764115 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764094.012731:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bde00 x1874599832690048/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764094.012736:0:257563:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764094.012744:0:257563:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc000 x1874599832690304/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764115 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764094.012747:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764094.012751:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764094.012757:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764094.012758:0:257563:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764094.012761:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764094.012762:0:257563:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787764094.012772:0:257563:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000100:00100000:1.0:1787764094.012773:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bde00 x1874599832690048/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:2.0:1787764094.012778:0:257563:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1e00.
00000400:00000200:2.0:1787764094.012785:0:257563:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00000001:1.0:1787764094.012791:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:2.0:1787764094.012794:0:257563:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787764094.012799:0:257563:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933110
00000100:00000040:1.0:1787764094.012802:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bde00 x1874599832690048/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1787764094.012803:0:257563:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933110.
00000100:00000001:2.0:1787764094.012808:0:257563:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:1.0:1787764094.012817:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000200:2.0:1787764094.012817:0:257563:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f87bed00 x1874599832690432/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764115 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764094.012825:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764094.012830:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764094.012833:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764094.012839:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764094.012843:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764094.012844:0:257563:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764094.012850:0:257563:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bed00 x1874599832690432/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764115 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787764094.012852:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764094.012863:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764094.012863:0:257563:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764094.012867:0:257563:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764094.012870:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764094.012878:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bde00 x1874599832690048/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764094.012888:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832690048:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787764094.012893:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764094.012895:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764094.012899:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bde00 x1874599832690048/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764094.012906:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764094.012910:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764094.012914:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c8000.
02000000:00000001:1.0:1787764094.012920:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764094.012923:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764094.012926:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787764094.012928:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764094.012933:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611c00.
02000000:00000001:1.0:1787764094.012938:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764094.012940:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764094.012943:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bde00.
00000100:00000001:1.0:1787764094.012947:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764094.012948:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764094.012951:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764094.012953:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764094.012955:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764094.012956:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764094.012961:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764090/real 1787764094]  req@ffff9751f87bed00 x1874599832690432/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764094.012970:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764094.012971:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764094.012974:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210000.
00000400:00000200:1.0:1787764094.012977:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933550
00000400:00000010:1.0:1787764094.012978:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933550.
00000100:00000001:1.0:1787764094.012981:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764094.012985:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bed00 x1874599832690432/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764094.012993:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bed00 x1874599832690432/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764094.012999:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764094.013001:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764094.013002:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764094.013004:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764094.013007:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bed00 x1874599832690432/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764094.013013:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764094.013017:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bed00 x1874599832690432/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764094.013024:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764094.013026:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764094.013027:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764094.013029:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764094.013031:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764094.013033:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764094.013036:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764094.013040:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764094.013043:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764094.013046:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bed00 x1874599832690432/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764094.013053:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832690432:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764094.013058:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764094.013062:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764090/real 1787764094]  req@ffff9751f87bc000 x1874599832690304/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764094.013069:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764094.013070:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764094.013072:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210d10.
00000400:00000200:1.0:1787764094.013075:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764094.013077:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764094.013079:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764094.013082:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc000 x1874599832690304/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764094.013090:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc000 x1874599832690304/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764094.013097:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764094.013098:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764094.013100:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764094.013101:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764094.013105:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc000 x1874599832690304/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764094.013112:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764094.013116:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc000 x1874599832690304/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764094.013122:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764094.013125:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764094.013126:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764094.013127:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764094.013130:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764094.013131:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764094.013176:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764094.013181:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764094.013184:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764094.013188:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc000 x1874599832690304/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764094.013196:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832690304:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764094.013200:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764094.013204:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764090/real 1787764094]  req@ffff9751f87be1c0 x1874599832690176/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764094.013211:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764094.013212:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764094.013214:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764094.013216:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764094.013218:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764094.013220:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764094.013223:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87be1c0 x1874599832690176/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764094.013230:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87be1c0 x1874599832690176/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764094.013237:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764094.013238:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764094.013240:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764094.013242:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764094.013246:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87be1c0 x1874599832690176/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764094.013252:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764094.013255:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87be1c0 x1874599832690176/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764094.013267:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764094.013270:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764094.013271:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764094.013272:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764094.013274:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764094.013288:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764094.013291:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764094.013295:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764094.013297:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764094.013301:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87be1c0 x1874599832690176/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764094.013309:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832690176:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764094.013314:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764094.013316:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764094.013319:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bed00 x1874599832690432/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764094.013326:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764094.013327:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764094.013329:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca000.
02000000:00000001:1.0:1787764094.013333:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764094.013334:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764094.013336:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787764094.013338:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764094.013339:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612000.
02000000:00000001:1.0:1787764094.013343:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764094.013350:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764094.013352:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bed00.
00000100:00000001:1.0:1787764094.013355:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764094.013356:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764094.013359:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764094.013362:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc000 x1874599832690304/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764094.013369:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764094.013371:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764094.013372:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764094.013375:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764094.013376:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764094.013378:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764094.013380:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764094.013381:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610800.
02000000:00000001:1.0:1787764094.013384:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764094.013385:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764094.013386:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc000.
00000100:00000001:1.0:1787764094.013389:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764094.013390:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764094.013392:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764094.013396:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87be1c0 x1874599832690176/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764115 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764094.013403:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764094.013404:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764094.013406:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9000.
02000000:00000001:1.0:1787764094.013409:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764094.013411:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764094.013412:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764094.013415:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764094.013416:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610c00.
02000000:00000001:1.0:1787764094.013419:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764094.013420:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764094.013422:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87be1c0.
00000100:00000001:1.0:1787764094.013424:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764094.013425:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764094.013427:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764094.013436:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764094.013437:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764094.013439:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764094.013442:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764094.652223:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764094.652232:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764094.652235:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764094.652239:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764094.652244:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764094.652247:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764094.652249:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764094.652252:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764094.652257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764094.652259:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764094.652263:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764094.652265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764094.652298:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764094.652304:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764094.652308:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764094.652312:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764094.652317:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764094.652321:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764094.652324:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764094.652328:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1787764094.652329:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000100:00000001:2.0:1787764094.652332:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764094.652334:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764094.652339:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764095.036255:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764095.036261:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764095.036263:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764095.036267:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764095.036270:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764095.036272:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764095.036274:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764095.036276:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764095.036278:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764095.676236:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764095.676243:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764095.676243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764095.676246:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764095.676249:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764095.676253:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764095.676257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764095.676258:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764095.676262:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764095.676263:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764095.676268:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764095.676270:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764095.676270:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764095.676275:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764095.676276:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764095.676357:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764095.676363:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764095.676384:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787764096.060221:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764096.060229:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764096.060232:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764096.060235:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764096.060239:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764096.060246:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21810
00000400:00000200:3.0:1787764096.060254:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764096.060261:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764096.060264:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764096.060267:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764096.060270:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764096.060272:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764096.060274:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764096.060278:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000020:00000001:3.0:1787764096.060278:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764096.060279:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787764096.060283:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1787764096.060284:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764096.060288:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1787764096.060288:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787764096.060293:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00100000:2.0:1787764096.060295:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000020:00000001:3.0:1787764096.060296:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000001:2.0:1787764096.060301:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:3.0:1787764096.060301:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764096.060305:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764096.060308:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764096.060308:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764096.060313:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764096.060315:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:3.0:1787764096.060316:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787764096.060320:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764096.060321:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764096.060323:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764096.060328:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764096.060328:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764096.060331:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764096.060335:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764096.060337:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764096.060340:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764096.060347:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bf480.
00000020:00000040:3.0:1787764096.060351:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787764096.060355:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764096.060356:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764096.060359:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764096.060362:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764096.060367:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764096.060375:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610400.
00000100:00000001:3.0:1787764096.060381:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764096.060392:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bf480 x1874599832690560/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764096.060403:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764096.060408:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764096.060413:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c2965388] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bf480 x1874599832690560/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764096.060442:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764096.060446:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764096.060455:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787764096.060457:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764096.060459:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764096.060462:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764096.060466:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21810
00000400:00000200:3.0:1787764096.060470:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764096.060475:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764096.060477:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764096.060482:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764096.060483:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787764096.060486:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1787764096.060486:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764096.060487:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764096.060492:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764096.060493:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00000001:1.0:1787764096.060494:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1787764096.060495:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764096.060499:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000001:1.0:1787764096.060500:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1787764096.060501:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:3.0:1787764096.060504:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:3.0:1787764096.060506:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764096.060509:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764096.060512:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000040:1.0:1787764096.060513:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bf480 x1874599832690560/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764096.060521:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787764096.060527:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764096.060531:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1787764096.060531:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:1.0:1787764096.060540:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:3.0:1787764096.060540:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:1.0:1787764096.060545:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764096.060546:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764096.060552:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764096.060552:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:1.0:1787764096.060560:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832690560:192.168.203.110@tcp:250:kworker.0
02000000:00000001:3.0:1787764096.060560:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764096.060578:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764096.060580:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764096.060585:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:3.0:1787764096.060585:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1787764096.060590:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1787764096.060592:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1787764096.060594:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:3.0:1787764096.060598:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764096.060602:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764096.060603:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9000.
02000000:00000001:1.0:1787764096.060607:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764096.060608:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87be1c0.
00000020:00000040:3.0:1787764096.060613:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787764096.060616:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764096.060618:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764096.060621:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764096.060622:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764096.060626:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:1.0:1787764096.060629:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
02000000:00000001:3.0:1787764096.060630:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764096.060639:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611800.
00000400:00000010:1.0:1787764096.060641:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000001:3.0:1787764096.060644:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787764096.060650:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832690560, portal 25
00000100:00100000:3.0:1787764096.060654:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87be1c0 x1874599832690688/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764096.060655:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764096.060659:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049859200 : -115096659692416 : ffff9751f87bf480)
00000100:00000001:3.0:1787764096.060669:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:1.0:1787764096.060671:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bf480 x1874599832690560/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764121 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764096.060673:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764096.060683:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e2011c6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87be1c0 x1874599832690688/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.060685:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764096.060693:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764096.060698:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000001:3.0:1787764096.060701:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787764096.060704:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832690560, offset 0
00000100:00080000:3.0:1787764096.060706:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764096.060712:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1787764096.060714:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764096.060717:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1787764096.060718:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787764096.060725:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764096.060729:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21810
00000400:00000200:1.0:1787764096.060732:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764096.060737:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764096.060742:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764096.060744:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764096.060749:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1787764096.060751:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:3.0:1787764096.060754:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764096.060759:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787764096.060760:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000040:3.0:1787764096.060764:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:1.0:1787764096.060768:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:3.0:1787764096.060770:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:1.0:1787764096.060775:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000020:00000001:3.0:1787764096.060776:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764096.060780:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000400:00000200:1.0:1787764096.060783:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58630:58630:256:4294967295] 192.168.203.110@tcp LPNI seq info [58630:58630:8:4294967295]
00000020:00000001:3.0:1787764096.060784:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764096.060788:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000400:00000200:1.0:1787764096.060794:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:3.0:1787764096.060794:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:3.0:1787764096.060800:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:3.0:1787764096.060806:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764096.060811:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787764096.060814:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:3.0:1787764096.060817:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764096.060822:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:1.0:1787764096.060828:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000020:00000040:3.0:1787764096.060828:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:3.0:1787764096.060836:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000800:00000010:1.0:1787764096.060838:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000100:00000001:3.0:1787764096.060840:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764096.060844:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764096.060848:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:1.0:1787764096.060849:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:3.0:1787764096.060852:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787764096.060856:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:3.0:1787764096.060857:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787764096.060861:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787764096.060866:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787764096.060867:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787764096.060871:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764096.060876:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc000.
00000020:00000040:3.0:1787764096.060881:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000800:00000200:1.1:1787764096.060881:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
02000000:00000001:3.0:1787764096.060886:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764096.060889:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:1.0:1787764096.060891:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787764096.060893:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000010:1.0:1787764096.060896:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
02000000:00000001:3.0:1787764096.060897:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764096.060901:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764096.060905:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000010:3.0:1787764096.060907:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613c00.
00000400:00000200:1.0:1787764096.060913:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:3.0:1787764096.060914:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764096.060924:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000100:00100000:3.0:1787764096.060924:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc000 x1874599832690816/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1787764096.060930:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764096.060937:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:3.0:1787764096.060938:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764096.060943:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787764096.060948:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bf480 x1874599832690560/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764121 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787764096.060950:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3fc1c6c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc000 x1874599832690816/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.060974:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764096.060979:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bf480 x1874599832690560/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764121 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.060987:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764096.060988:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764096.060996:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764096.061000:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764096.061003:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316484339
00000100:00000010:1.0:1787764096.061008:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4840.
00000100:00000001:1.0:1787764096.061015:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787764096.061022:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316484339
00000100:00000010:1.0:1787764096.061027:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4720.
00000100:00000001:1.0:1787764096.061030:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787764096.061033:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764096.061040:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764096.061043:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764096.061044:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000040:3.0:1787764096.061049:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:1.0:1787764096.061053:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764096/real 1787764096]  req@ffff9751f87bf480 x1874599832690560/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787764096.061056:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:3.0:1787764096.061062:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764096.061071:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787764096.061072:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764096.061074:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764096.061077:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764096.061082:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21810
00000400:00000001:1.0:1787764096.061087:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000200:3.0:1787764096.061088:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000001:1.0:1787764096.061092:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000200:3.0:1787764096.061094:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000010:1.0:1787764096.061098:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000100:00080000:3.0:1787764096.061098:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764096.061103:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:1.0:1787764096.061106:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000100:00000001:3.0:1787764096.061107:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:1.0:1787764096.061111:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000040:3.0:1787764096.061112:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764096.061117:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787764096.061118:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000001:3.0:1787764096.061121:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764096.061125:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000100:00000200:1.0:1787764096.061127:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bf480 x1874599832690560/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787764096.061130:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764096.061175:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:3.0:1787764096.061180:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000200:1.0:1787764096.061181:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bf480 x1874599832690560/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787764096.061185:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:3.0:1787764096.061194:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764096.061197:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764096.061198:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787764096.061202:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764096.061203:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764096.061208:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787764096.061209:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000040:3.0:1787764096.061213:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:1.0:1787764096.061214:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1787764096.061218:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764096.061223:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764096.061225:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bf480 x1874599832690560/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787764096.061226:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764096.061231:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764096.061234:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764096.061236:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787764096.061239:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1787764096.061241:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764096.061243:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764096.061248:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bed00.
00000020:00000040:3.0:1787764096.061252:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:1.0:1787764096.061256:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787764096.061258:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764096.061261:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764096.061264:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764096.061267:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:1.0:1787764096.061268:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bf480 x1874599832690560/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787764096.061270:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764096.061278:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613800.
00000100:00000001:1.0:1787764096.061282:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764096.061285:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764096.061289:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764096.061295:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00100000:3.0:1787764096.061296:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bed00 x1874599832690944/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764096.061297:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764096.061304:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764096.061308:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764096.061310:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00080000:1.0:1787764096.061315:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:3.0:1787764096.061316:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764096.061326:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000079751c9a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bed00 x1874599832690944/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787764096.061327:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764096.061333:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:3.0:1787764096.061342:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764096.061344:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bf480 x1874599832690560/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787764096.061348:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21820)
00000100:00100000:1.0:1787764096.061360:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832690560:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787764096.061365:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764096.061368:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764096.061372:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bf480 x1874599832690560/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.061379:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764096.061382:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764096.061385:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9000.
02000000:00000001:1.0:1787764096.061388:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764096.061390:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764096.061391:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787764096.061393:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764096.061396:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610400.
02000000:00000001:1.0:1787764096.061399:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764096.061400:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764096.061402:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bf480.
00000100:00000001:1.0:1787764096.061405:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764096.061406:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764096.061408:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764096.061416:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764096.061417:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764096.061419:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764096.061421:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764096.061423:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764096.061427:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87be1c0 x1874599832690688/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.061440:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764096.061442:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764096.061445:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764096.061447:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764096.061449:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764096.061453:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832690688:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764096.061456:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764096.061459:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764096.061461:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764096.061463:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764096.061467:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9000.
02000000:00000001:1.0:1787764096.061469:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764096.061474:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764096.061477:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764096.061482:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764096.061486:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832690688, portal 10
00000100:00000001:1.0:1787764096.061488:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764096.061489:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049854400 : -115096659697216 : ffff9751f87be1c0)
00000100:00000040:1.0:1787764096.061496:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87be1c0 x1874599832690688/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764121 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.061503:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764096.061505:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764096.061508:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764096.061511:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832690688, offset 0
00000400:00000200:1.0:1787764096.061515:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764096.061520:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764096.061524:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764096.061528:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764096.061531:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764096.061534:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764096.061537:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764096.061542:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58631:58631:256:4294967295] 192.168.203.110@tcp LPNI seq info [58631:58631:8:4294967295]
00000400:00000200:1.0:1787764096.061547:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764096.061554:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764096.061559:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764096.061574:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764096.061580:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764096.061583:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787764096.061588:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764096.061591:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1787764096.061595:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764096.061597:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764096.061604:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764096.061609:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764096.061613:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764096.061615:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764096.061618:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764096.061622:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87be1c0 x1874599832690688/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764121 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.061631:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764096.061634:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87be1c0 x1874599832690688/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764121 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.061642:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764096.061643:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764096.061647:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764096.061651:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787764096.061654:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764096.061656:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316483104
00000100:00000001:1.0:1787764096.061659:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316484339
00000100:00000001:1.0:1787764096.061661:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316483104
00000100:00000001:1.0:1787764096.061663:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316484339
00000100:00000001:1.0:1787764096.061665:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764096.061668:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764096.061670:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764096.061674:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764096/real 1787764096]  req@ffff9751f87be1c0 x1874599832690688/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764096.061683:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764096.061684:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764096.061686:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764096.061688:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764096.061690:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764096.061692:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764096.061695:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87be1c0 x1874599832690688/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764096.061704:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87be1c0 x1874599832690688/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.061710:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764096.061712:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764096.061714:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764096.061715:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764096.061719:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87be1c0 x1874599832690688/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.061726:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764096.061729:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87be1c0 x1874599832690688/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.061734:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764096.061737:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764096.061739:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764096.061740:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764096.061742:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764096.061758:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764096.061761:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764096.061766:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764096.061768:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764096.061772:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87be1c0 x1874599832690688/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764096.061781:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832690688:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764096.061784:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764096.061787:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc000 x1874599832690816/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.061794:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764096.061795:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764096.061798:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764096.061800:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764096.061802:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764096.061805:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832690816:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764096.061808:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764096.061811:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764096.061813:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764096.061815:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764096.061819:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764096.061820:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764096.061823:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764096.061825:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764096.061827:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764096.061829:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832690816, portal 10
00000100:00000001:1.0:1787764096.061831:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764096.061832:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049845760 : -115096659705856 : ffff9751f87bc000)
00000100:00000040:1.0:1787764096.061836:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc000 x1874599832690816/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764121 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.061841:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764096.061843:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764096.061844:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764096.061846:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832690816, offset 0
00000400:00000200:1.0:1787764096.061848:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764096.061851:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764096.061854:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764096.061856:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764096.061858:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764096.061860:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764096.061861:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764096.061863:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58632:58632:256:4294967295] 192.168.203.110@tcp LPNI seq info [58632:58632:8:4294967295]
00000400:00000200:1.0:1787764096.061866:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764096.061870:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764096.061873:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764096.061875:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764096.061878:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764096.061880:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787764096.061882:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764096.061884:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1787764096.061887:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764096.061888:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764096.061890:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764096.061893:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764096.061895:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764096.061896:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764096.061898:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764096.061900:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc000 x1874599832690816/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764121 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.061905:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764096.061907:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc000 x1874599832690816/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764121 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.061911:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764096.061912:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764096.061915:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764096.061917:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787764096.061919:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764096.061920:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316483092
00000100:00000001:1.0:1787764096.061922:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316484339
00000100:00000001:1.0:1787764096.061923:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316483092
00000100:00000001:1.0:1787764096.061925:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316484339
00000100:00000001:1.0:1787764096.061926:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764096.061927:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764096.061928:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764096.061931:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764096/real 1787764096]  req@ffff9751f87bc000 x1874599832690816/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764096.061936:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764096.061936:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764096.061938:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764096.061939:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764096.061940:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764096.061942:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764096.061943:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc000 x1874599832690816/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764096.061948:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc000 x1874599832690816/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.061952:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764096.061953:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764096.061955:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764096.061955:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764096.061958:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc000 x1874599832690816/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.061962:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764096.061964:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc000 x1874599832690816/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.061969:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764096.061971:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764096.061972:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764096.061973:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764096.061974:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764096.061975:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764096.061977:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764096.061978:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764096.061980:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764096.061982:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc000 x1874599832690816/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764096.061987:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832690816:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764096.061989:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764096.061991:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bed00 x1874599832690944/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.061995:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764096.061996:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764096.061998:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764096.061999:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764096.062000:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764096.062003:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832690944:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764096.062006:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764096.062008:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764096.062010:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764096.062011:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764096.062015:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca000.
02000000:00000001:1.0:1787764096.062017:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764096.062020:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764096.062022:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764096.062026:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764096.062028:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832690944, portal 4
00000100:00000001:1.0:1787764096.062031:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764096.062032:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049857280 : -115096659694336 : ffff9751f87bed00)
00000100:00000040:1.0:1787764096.062038:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bed00 x1874599832690944/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764121 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.062045:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764096.062047:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764096.062050:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764096.062052:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832690944, offset 0
00000400:00000200:1.0:1787764096.062056:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764096.062060:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764096.062063:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764096.062066:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764096.062069:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764096.062072:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764096.062074:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764096.062077:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58633:58633:256:4294967295] 192.168.203.110@tcp LPNI seq info [58633:58633:8:4294967295]
00000400:00000200:1.0:1787764096.062082:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764096.062088:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764096.062092:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764096.062095:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764096.062097:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764096.062099:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787764096.062102:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764096.062104:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1787764096.062106:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764096.062108:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764096.062110:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764096.062112:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764096.062114:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764096.062116:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764096.062118:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764096.062120:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bed00 x1874599832690944/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764121 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.062125:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764096.062127:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bed00 x1874599832690944/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764121 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.062132:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764096.062151:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764096.062154:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764096.062156:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787764096.062159:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764096.062160:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316483110
00000100:00000001:1.0:1787764096.062162:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316484339
00000100:00000001:1.0:1787764096.062163:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316483110
00000100:00000001:1.0:1787764096.062165:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316484339
00000100:00000001:1.0:1787764096.062167:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764096.062168:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764096.062169:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764096.062172:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764096/real 1787764096]  req@ffff9751f87bed00 x1874599832690944/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764096.062176:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764096.062177:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764096.062178:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764096.062180:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764096.062181:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764096.062182:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764096.062184:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bed00 x1874599832690944/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764096.062189:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bed00 x1874599832690944/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.062194:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764096.062195:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764096.062196:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764096.062197:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764096.062200:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bed00 x1874599832690944/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.062204:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764096.062206:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bed00 x1874599832690944/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.062210:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764096.062211:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764096.062212:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764096.062213:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764096.062214:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764096.062215:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764096.062217:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764096.062219:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764096.062221:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764096.062223:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bed00 x1874599832690944/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764096.062231:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832690944:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764096.062234:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764096.062236:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764096.062240:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87be1c0 x1874599832690688/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.062246:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764096.062248:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764096.062251:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9000.
02000000:00000001:1.0:1787764096.062254:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764096.062256:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764096.062257:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764096.062260:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764096.062262:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611800.
02000000:00000001:1.0:1787764096.062265:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764096.062266:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764096.062268:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87be1c0.
00000100:00000001:1.0:1787764096.062270:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764096.062272:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764096.062274:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764096.062277:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc000 x1874599832690816/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.062285:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764096.062286:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764096.062287:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764096.062290:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764096.062291:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764096.062292:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764096.062294:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764096.062295:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613c00.
02000000:00000001:1.0:1787764096.062298:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764096.062299:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764096.062300:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc000.
00000100:00000001:1.0:1787764096.062308:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764096.062310:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764096.062311:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764096.062315:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bed00 x1874599832690944/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764121 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764096.062321:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764096.062322:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764096.062324:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca000.
02000000:00000001:1.0:1787764096.062326:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764096.062327:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764096.062328:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787764096.062330:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764096.062332:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613800.
02000000:00000001:1.0:1787764096.062334:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764096.062335:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764096.062336:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bed00.
00000100:00000001:1.0:1787764096.062338:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764096.062339:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764096.062341:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764096.062346:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764096.062347:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764096.062349:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764096.062351:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764096.700235:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764096.700238:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764096.700243:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764096.700243:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764096.700247:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764096.700249:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764096.700251:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764096.700252:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764096.700255:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764096.700258:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764096.700259:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764096.700261:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764096.700262:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764096.700265:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764096.700268:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764096.700329:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764096.700334:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764096.700338:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764097.084308:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764097.084314:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764097.084316:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764097.084319:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764097.084322:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764097.084324:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764097.084326:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764097.084328:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764097.084330:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764097.340220:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316484339, at 4316484352
00000100:00000010:3.0:1787764097.340245:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4720.
00000100:00000040:2.0:1787764097.340261:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316484339, at 4316484352
00000100:00000010:2.0:1787764097.340269:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4840.
00000100:00000001:2.0:1787764097.724285:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764097.724285:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764097.724293:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764097.724295:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764097.724301:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764097.724303:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764097.724306:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764097.724309:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764097.724313:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764097.724314:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764097.724320:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764097.724325:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764097.724328:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764097.724331:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764097.724332:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764097.724340:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764097.724344:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764097.724347:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764097.724350:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764097.724355:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764097.724357:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764097.724502:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764097.724508:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764097.724513:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764098.108204:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764098.108210:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764098.108212:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764098.108217:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764098.108220:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764098.108223:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764098.108225:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764098.108228:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764098.108230:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764098.748234:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764098.748258:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764098.748259:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764098.748264:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764098.748267:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764098.748268:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764098.748270:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764098.748272:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764098.748273:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764098.748281:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764098.748286:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764098.748290:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764098.748293:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764098.748297:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764098.748301:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764098.748358:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764098.748362:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764098.748365:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764099.132212:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764099.132218:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764099.132220:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764099.132225:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764099.132228:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764099.132231:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764099.132233:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764099.132235:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764099.132237:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764099.772365:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764099.772378:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764099.772383:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764099.772392:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764099.772399:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764099.772421:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764099.772423:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764099.772430:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764099.772434:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764099.772440:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764099.772445:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764099.772446:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764099.772459:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764099.772461:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764099.772463:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764099.772616:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764099.772623:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764099.772630:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764100.156316:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764100.156327:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764100.156330:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764100.156338:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764100.156344:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764100.156360:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764100.156365:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764100.156369:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764100.156372:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764100.796257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764100.796258:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764100.796264:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764100.796288:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764100.796289:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764100.796292:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764100.796297:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764100.796300:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764100.796305:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764100.796311:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764100.796317:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764100.796324:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764100.796328:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764100.796333:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764100.796338:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764100.796343:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764100.796351:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764100.796356:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764101.180236:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1787764101.180236:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764101.180244:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:3.0:1787764101.180245:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787764101.180249:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764101.180249:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764101.180254:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1787764101.180256:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764101.180258:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764101.180266:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21815
00000100:00000001:2.0:1787764101.180273:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000400:00000200:3.0:1787764101.180275:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764101.180280:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:3.0:1787764101.180283:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787764101.180287:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:3.0:1787764101.180288:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787764101.180292:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764101.180293:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764101.180299:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764101.180300:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764101.180303:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764101.180307:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764101.180310:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764101.180311:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787764101.180313:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764101.180316:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787764101.180318:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787764101.180320:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787764101.180323:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764101.180324:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764101.180326:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764101.180328:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764101.180331:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787764101.180333:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764101.180335:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764101.180337:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764101.180340:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764101.180342:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764101.180343:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764101.180345:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764101.180353:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bde00.
00000020:00000040:3.0:1787764101.180355:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787764101.180358:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764101.180359:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764101.180361:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764101.180363:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764101.180367:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764101.180373:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612400.
00000100:00000001:3.0:1787764101.180378:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764101.180385:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bde00 x1874599832691072/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764101.180393:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764101.180397:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764101.180400:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000941163c2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bde00 x1874599832691072/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764101.180415:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764101.180417:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764101.180419:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787764101.180420:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764101.180422:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764101.180424:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764101.180426:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21815
00000400:00000200:3.0:1787764101.180430:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764101.180433:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764101.180435:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787764101.180436:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1787764101.180438:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764101.180441:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764101.180441:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764101.180443:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764101.180445:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1787764101.180447:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764101.180449:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764101.180451:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:3.0:1787764101.180453:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:1.0:1787764101.180454:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bde00 x1874599832691072/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787764101.180455:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:3.0:1787764101.180459:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:3.0:1787764101.180463:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:3.0:1787764101.180470:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764101.180471:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764101.180474:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764101.180475:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764101.180479:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:1.0:1787764101.180480:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1787764101.180481:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:1.0:1787764101.180484:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:3.0:1787764101.180485:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:1.0:1787764101.180490:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764101.180491:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764101.180495:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764101.180496:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832691072:192.168.203.110@tcp:250:kworker.0
02000000:00000001:3.0:1787764101.180497:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787764101.180502:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764101.180503:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764101.180505:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764101.180507:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764101.180508:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764101.180511:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1787764101.180512:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:3.0:1787764101.180515:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000010:1.0:1787764101.180518:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca000.
00000008:00000001:3.0:1787764101.180519:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764101.180523:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764101.180524:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764101.180547:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bfc00.
00000400:00000200:1.0:1787764101.180551:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:3.0:1787764101.180551:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787764101.180556:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764101.180557:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764101.180559:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:1.0:1787764101.180560:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
02000000:00000001:3.0:1787764101.180560:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764101.180564:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764101.180569:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612c00.
00000400:00000010:1.0:1787764101.180570:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000001:3.0:1787764101.180573:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787764101.180577:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832691072, portal 25
00000100:00100000:3.0:1787764101.180580:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bfc00 x1874599832691200/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764101.180581:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764101.180584:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049853440 : -115096659698176 : ffff9751f87bde00)
00000100:00000001:3.0:1787764101.180592:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:1.0:1787764101.180594:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bde00 x1874599832691072/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764126 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764101.180595:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764101.180603:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003c1f6b31] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bfc00 x1874599832691200/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.180608:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764101.180614:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764101.180618:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000001:3.0:1787764101.180618:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764101.180621:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764101.180624:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000200:1.0:1787764101.180625:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832691072, offset 0
00000100:00000001:3.0:1787764101.180626:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764101.180628:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764101.180631:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764101.180633:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21815
00000400:00000200:1.0:1787764101.180638:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764101.180638:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764101.180643:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764101.180645:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764101.180647:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764101.180649:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764101.180652:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:1.0:1787764101.180653:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764101.180654:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764101.180658:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764101.180659:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:3.0:1787764101.180660:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764101.180661:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:3.0:1787764101.180663:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:1.0:1787764101.180665:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000020:00000001:3.0:1787764101.180666:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:3.0:1787764101.180671:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:1.0:1787764101.180672:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1787764101.180673:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764101.180676:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:1.0:1787764101.180679:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:3.0:1787764101.180680:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764101.180683:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:1.0:1787764101.180684:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00080000:3.0:1787764101.180687:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:1.0:1787764101.180691:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:3.0:1787764101.180692:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764101.180695:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787764101.180698:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58634:58634:256:4294967295] 192.168.203.110@tcp LPNI seq info [58634:58634:8:4294967295]
02000000:00000001:3.0:1787764101.180698:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764101.180703:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764101.180707:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764101.180707:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787764101.180711:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787764101.180714:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764101.180716:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764101.180721:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000010:3.0:1787764101.180722:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc3c0.
00000020:00000040:3.0:1787764101.180726:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787764101.180729:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764101.180732:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787764101.180733:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787764101.180734:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764101.180736:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764101.180738:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1787764101.180739:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
02000000:00000010:3.0:1787764101.180742:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612800.
00000100:00000001:3.0:1787764101.180745:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787764101.180750:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00100000:3.0:1787764101.180754:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc3c0 x1874599832691328/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:1.0:1787764101.180756:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:3.0:1787764101.180765:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:1.1:1787764101.180766:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787764101.180768:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787764101.180772:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000100:00000040:3.0:1787764101.180775:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000060bf8470] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc3c0 x1874599832691328/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:1.0:1787764101.180781:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764101.180785:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000100:00000001:3.0:1787764101.180788:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764101.180791:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1787764101.180792:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1787764101.180798:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:1.0:1787764101.180801:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:3.0:1787764101.180802:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764101.180806:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000200:1.0:1787764101.180807:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000100:00000001:3.0:1787764101.180808:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:1.0:1787764101.180812:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00080000:3.0:1787764101.180812:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764101.180817:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1787764101.180819:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:3.0:1787764101.180820:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21815
00000400:00000200:3.0:1787764101.180826:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000200:1.0:1787764101.180827:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bde00 x1874599832691072/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764126 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764101.180830:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764101.180834:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764101.180840:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764101.180844:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787764101.180847:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764101.180847:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764101.180851:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:1.0:1787764101.180855:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bde00 x1874599832691072/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764126 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787764101.180855:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764101.180857:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:3.0:1787764101.180860:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764101.180863:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:3.0:1787764101.180867:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:1.0:1787764101.180869:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787764101.180870:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:1.0:1787764101.180872:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:3.0:1787764101.180875:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764101.180877:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787764101.180879:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764101.180879:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764101.180881:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764101.180884:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000400:00000200:1.0:1787764101.180887:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:3.0:1787764101.180890:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764101.180891:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316489459
00000100:00000001:3.0:1787764101.180893:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764101.180896:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000010:1.0:1787764101.180897:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c45a0.
02000000:00000001:3.0:1787764101.180898:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764101.180902:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764101.180903:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787764101.180905:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1787764101.180906:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000008:00000001:3.0:1787764101.180907:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764101.180911:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764101.180915:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316489459
00000100:00000010:3.0:1787764101.180915:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc780.
00000020:00000040:3.0:1787764101.180918:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:3.0:1787764101.180921:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000010:1.0:1787764101.180922:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4e40.
02000000:00000001:3.0:1787764101.180924:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764101.180926:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787764101.180927:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764101.180930:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1787764101.180933:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764101.180933:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764101.180938:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764101.180939:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613000.
00000100:00000001:1.0:1787764101.180941:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:3.0:1787764101.180943:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764101.180947:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764101/real 1787764101]  req@ffff9751f87bde00 x1874599832691072/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764101.180949:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc780 x1874599832691456/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764101.180979:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000001:1.0:1787764101.180980:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:3.0:1787764101.180981:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1787764101.180983:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:3.0:1787764101.180986:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7d56184] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc780 x1874599832691456/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1787764101.180987:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764101.180992:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764101.180995:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764101.180998:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:3.0:1787764101.181000:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787764101.181004:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bde00 x1874599832691072/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787764101.181005:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21825)
00000100:00000200:1.0:1787764101.181014:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bde00 x1874599832691072/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181020:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764101.181021:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764101.181023:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764101.181024:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764101.181027:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bde00 x1874599832691072/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181032:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764101.181035:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bde00 x1874599832691072/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181040:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764101.181041:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764101.181042:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764101.181043:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764101.181045:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764101.181046:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764101.181048:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764101.181050:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764101.181052:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764101.181055:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bde00 x1874599832691072/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764101.181060:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832691072:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787764101.181063:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764101.181064:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764101.181066:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bde00 x1874599832691072/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181071:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764101.181073:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764101.181075:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca000.
02000000:00000001:1.0:1787764101.181077:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764101.181078:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764101.181079:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787764101.181081:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764101.181083:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612400.
02000000:00000001:1.0:1787764101.181085:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764101.181087:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764101.181088:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bde00.
00000100:00000001:1.0:1787764101.181090:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764101.181091:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764101.181093:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764101.181098:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764101.181099:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764101.181100:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764101.181102:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764101.181102:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764101.181105:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bfc00 x1874599832691200/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181110:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764101.181111:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764101.181112:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764101.181114:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764101.181115:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764101.181117:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832691200:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764101.181119:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764101.181120:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764101.181121:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764101.181122:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764101.181124:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca000.
02000000:00000001:1.0:1787764101.181125:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764101.181128:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764101.181130:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764101.181149:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764101.181151:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832691200, portal 10
00000100:00000001:1.0:1787764101.181153:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764101.181154:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049861120 : -115096659690496 : ffff9751f87bfc00)
00000100:00000040:1.0:1787764101.181159:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bfc00 x1874599832691200/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764126 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181163:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764101.181165:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764101.181167:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764101.181169:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832691200, offset 0
00000400:00000200:1.0:1787764101.181172:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764101.181174:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764101.181177:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764101.181179:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764101.181181:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764101.181183:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764101.181184:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764101.181186:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58635:58635:256:4294967295] 192.168.203.110@tcp LPNI seq info [58635:58635:8:4294967295]
00000400:00000200:1.0:1787764101.181189:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764101.181194:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764101.181197:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764101.181199:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764101.181201:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764101.181203:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787764101.181206:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764101.181208:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1787764101.181211:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764101.181212:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764101.181214:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764101.181217:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764101.181219:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764101.181220:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764101.181222:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764101.181225:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bfc00 x1874599832691200/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764126 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181233:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764101.181236:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bfc00 x1874599832691200/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764126 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181243:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764101.181245:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764101.181249:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764101.181253:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787764101.181257:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764101.181259:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316488224
00000100:00000001:1.0:1787764101.181262:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316489459
00000100:00000001:1.0:1787764101.181265:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316488224
00000100:00000001:1.0:1787764101.181267:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316489459
00000100:00000001:1.0:1787764101.181269:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764101.181271:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764101.181273:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764101.181278:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764101/real 1787764101]  req@ffff9751f87bfc00 x1874599832691200/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764101.181285:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764101.181286:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764101.181287:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764101.181289:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764101.181290:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764101.181292:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764101.181294:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bfc00 x1874599832691200/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764101.181300:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bfc00 x1874599832691200/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181304:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764101.181305:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764101.181306:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764101.181307:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764101.181309:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bfc00 x1874599832691200/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181313:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764101.181315:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bfc00 x1874599832691200/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181321:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764101.181322:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764101.181323:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764101.181324:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764101.181325:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764101.181335:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764101.181337:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764101.181339:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764101.181340:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764101.181343:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bfc00 x1874599832691200/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764101.181348:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832691200:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764101.181350:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764101.181352:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc3c0 x1874599832691328/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181356:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764101.181357:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764101.181358:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764101.181359:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764101.181361:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764101.181362:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832691328:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764101.181364:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764101.181365:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764101.181367:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764101.181368:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764101.181370:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764101.181372:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764101.181374:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764101.181376:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764101.181378:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764101.181380:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832691328, portal 10
00000100:00000001:1.0:1787764101.181381:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764101.181382:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049846720 : -115096659704896 : ffff9751f87bc3c0)
00000100:00000040:1.0:1787764101.181386:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc3c0 x1874599832691328/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764126 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181390:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764101.181392:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764101.181393:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764101.181395:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832691328, offset 0
00000400:00000200:1.0:1787764101.181398:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764101.181400:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764101.181402:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764101.181404:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764101.181406:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764101.181408:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764101.181409:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764101.181411:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58636:58636:256:4294967295] 192.168.203.110@tcp LPNI seq info [58636:58636:8:4294967295]
00000400:00000200:1.0:1787764101.181414:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764101.181418:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764101.181421:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764101.181423:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764101.181425:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764101.181427:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787764101.181430:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764101.181432:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1787764101.181435:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764101.181436:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764101.181438:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764101.181440:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764101.181442:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764101.181444:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764101.181446:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764101.181448:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc3c0 x1874599832691328/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764126 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181453:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764101.181455:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc3c0 x1874599832691328/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764126 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181459:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764101.181460:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764101.181462:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764101.181464:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787764101.181466:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764101.181467:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316488212
00000100:00000001:1.0:1787764101.181468:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316489459
00000100:00000001:1.0:1787764101.181470:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316488212
00000100:00000001:1.0:1787764101.181472:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316489459
00000100:00000001:1.0:1787764101.181473:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764101.181474:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764101.181476:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764101.181478:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764101/real 1787764101]  req@ffff9751f87bc3c0 x1874599832691328/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764101.181482:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764101.181483:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764101.181484:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764101.181485:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764101.181486:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764101.181487:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764101.181489:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc3c0 x1874599832691328/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764101.181494:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc3c0 x1874599832691328/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181499:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764101.181500:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764101.181501:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764101.181502:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764101.181504:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc3c0 x1874599832691328/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181508:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764101.181510:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc3c0 x1874599832691328/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181514:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764101.181515:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764101.181516:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764101.181517:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764101.181518:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764101.181519:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764101.181521:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764101.181522:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764101.181524:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764101.181526:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc3c0 x1874599832691328/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764101.181547:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832691328:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764101.181549:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764101.181552:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc780 x1874599832691456/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181556:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764101.181557:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764101.181559:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764101.181560:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764101.181562:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764101.181563:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832691456:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764101.181566:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764101.181567:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764101.181568:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764101.181569:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764101.181572:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9000.
02000000:00000001:1.0:1787764101.181573:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764101.181576:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764101.181577:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764101.181580:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764101.181583:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832691456, portal 4
00000100:00000001:1.0:1787764101.181585:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764101.181586:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049847680 : -115096659703936 : ffff9751f87bc780)
00000100:00000040:1.0:1787764101.181592:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc780 x1874599832691456/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764126 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181598:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764101.181601:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764101.181603:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764101.181606:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832691456, offset 0
00000400:00000200:1.0:1787764101.181610:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764101.181614:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764101.181618:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764101.181621:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764101.181623:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764101.181625:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764101.181628:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764101.181631:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58637:58637:256:4294967295] 192.168.203.110@tcp LPNI seq info [58637:58637:8:4294967295]
00000400:00000200:1.0:1787764101.181635:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764101.181642:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764101.181647:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764101.181650:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764101.181655:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764101.181658:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:1.1:1787764101.181662:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764101.181665:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:1.0:1787764101.181670:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764101.181672:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764101.181675:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764101.181679:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764101.181681:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764101.181683:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764101.181684:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764101.181687:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc780 x1874599832691456/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764126 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181693:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764101.181695:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc780 x1874599832691456/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764126 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181699:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764101.181700:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764101.181702:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764101.181704:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787764101.181707:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764101.181708:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316488229
00000100:00000001:1.0:1787764101.181710:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316489459
00000100:00000001:1.0:1787764101.181711:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316488229
00000100:00000001:1.0:1787764101.181713:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316489459
00000100:00000001:1.0:1787764101.181714:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764101.181715:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764101.181716:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764101.181719:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764101/real 1787764101]  req@ffff9751f87bc780 x1874599832691456/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764101.181724:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764101.181725:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764101.181727:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764101.181728:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764101.181729:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764101.181730:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764101.181732:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc780 x1874599832691456/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764101.181738:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc780 x1874599832691456/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181741:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764101.181742:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764101.181743:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764101.181744:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764101.181746:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc780 x1874599832691456/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181750:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764101.181753:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc780 x1874599832691456/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181757:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764101.181758:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764101.181759:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764101.181760:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764101.181761:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764101.181763:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764101.181764:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764101.181766:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764101.181767:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764101.181770:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc780 x1874599832691456/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764101.181776:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832691456:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764101.181778:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764101.181779:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764101.181781:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bfc00 x1874599832691200/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181785:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764101.181786:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764101.181787:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca000.
02000000:00000001:1.0:1787764101.181789:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764101.181789:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764101.181790:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764101.181791:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764101.181793:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612c00.
02000000:00000001:1.0:1787764101.181794:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764101.181795:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764101.181796:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bfc00.
00000100:00000001:1.0:1787764101.181797:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764101.181798:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764101.181799:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764101.181801:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc3c0 x1874599832691328/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181805:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764101.181806:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764101.181807:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764101.181808:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764101.181809:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764101.181810:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764101.181811:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764101.181811:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612800.
02000000:00000001:1.0:1787764101.181812:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764101.181814:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764101.181814:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc3c0.
00000100:00000001:1.0:1787764101.181816:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764101.181817:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764101.181818:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764101.181820:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc780 x1874599832691456/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764126 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764101.181823:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764101.181824:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764101.181825:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9000.
02000000:00000001:1.0:1787764101.181826:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764101.181827:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764101.181828:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787764101.181829:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764101.181829:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613000.
02000000:00000001:1.0:1787764101.181831:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764101.181832:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764101.181832:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc780.
00000100:00000001:1.0:1787764101.181833:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764101.181834:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764101.181835:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764101.181838:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764101.181838:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764101.181840:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764101.181841:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764101.820248:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764101.820257:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764101.820259:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764101.820264:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764101.820265:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764101.820277:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764101.820277:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764101.820284:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764101.820285:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764101.820291:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764101.820291:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764101.820301:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764101.820303:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764101.820305:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764101.820307:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764101.820311:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764101.820314:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764101.820316:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764101.820318:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764101.820322:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764101.820323:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764101.820389:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764101.820396:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764101.820399:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764102.204246:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764102.204256:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764102.204258:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764102.204263:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764102.204267:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764102.204271:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764102.204274:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764102.204276:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764102.204279:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764102.460245:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316489459, at 4316489472
00000100:00000010:3.0:1787764102.460264:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4e40.
00000100:00000040:2.0:1787764102.460314:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316489459, at 4316489472
00000100:00000010:2.0:1787764102.460330:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c45a0.
00000100:00000001:2.0:1787764102.844219:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764102.844227:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764102.844230:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764102.844236:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764102.844241:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764102.844246:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764102.844249:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764102.844252:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764102.844255:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764102.844260:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764102.844265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764102.844271:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764102.844272:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764102.844275:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764102.844276:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764102.844339:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764102.844343:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764102.844347:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764103.228305:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764103.228314:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764103.228316:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764103.228322:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764103.228328:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764103.228332:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764103.228335:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764103.228339:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764103.228342:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764103.868307:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764103.868317:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764103.868324:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764103.868328:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764103.868333:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764103.868334:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764103.868340:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764103.868344:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764103.868348:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764103.868349:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764103.868356:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764103.868360:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764103.868364:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764103.868367:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764103.868370:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764103.868445:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764103.868450:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764103.868455:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764104.252210:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764104.252217:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764104.252219:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764104.252224:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764104.252228:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764104.252232:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764104.252234:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764104.252236:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764104.252238:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764104.892282:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764104.892283:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764104.892310:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764104.892313:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764104.892316:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764104.892322:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764104.892331:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764104.892332:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764104.892337:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764104.892339:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764104.892344:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764104.892348:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764104.892349:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764104.892375:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764104.892381:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764104.892385:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764104.892385:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764104.892398:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764104.892400:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764104.892421:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764104.892425:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764104.892519:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764104.892525:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764104.892529:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764105.276383:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764105.276392:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764105.276395:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764105.276401:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764105.276406:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764105.276409:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764105.276412:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764105.276414:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764105.276418:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764105.916255:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764105.916260:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764105.916264:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764105.916266:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764105.916268:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764105.916277:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764105.916278:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764105.916283:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764105.916286:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764105.916289:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764105.916294:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764105.916295:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764105.916303:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764105.916307:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764105.916311:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764105.916412:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764105.916418:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764105.916422:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787764106.300236:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787764106.300242:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1787764106.300253:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787764106.300254:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764106.300264:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764106.300265:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764106.300274:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:1.0:1787764106.300275:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764106.300285:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1787764106.300286:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764106.300292:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764106.300296:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764106.300301:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1787764106.300301:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21820
00000100:00000001:1.0:1787764106.300307:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764106.300319:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764106.300330:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764106.300335:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764106.300339:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764106.300344:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764106.300346:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764106.300348:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764106.300353:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764106.300355:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787764106.300358:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764106.300362:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787764106.300364:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787764106.300366:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787764106.300368:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764106.300369:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764106.300370:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764106.300372:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764106.300374:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787764106.300376:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764106.300378:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764106.300381:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764106.300384:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764106.300387:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764106.300388:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764106.300391:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764106.300401:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bcf00.
00000020:00000040:3.0:1787764106.300404:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787764106.300409:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764106.300410:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764106.300413:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764106.300416:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764106.300420:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764106.300428:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611400.
00000100:00000001:3.0:1787764106.300435:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764106.300444:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bcf00 x1874599832691584/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764106.300453:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764106.300458:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764106.300462:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ebc1bc5] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bcf00 x1874599832691584/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764106.300486:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764106.300489:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764106.300504:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787764106.300506:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764106.300509:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764106.300519:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764106.300520:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1787764106.300524:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764106.300527:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00080000:3.0:1787764106.300528:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21820
00000100:00000040:1.0:1787764106.300541:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bcf00 x1874599832691584/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764106.300550:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764106.300566:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000400:00000200:3.0:1787764106.300566:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1787764106.300573:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764106.300575:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
02000000:00000001:1.0:1787764106.300578:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:3.0:1787764106.300584:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:1.0:1787764106.300586:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764106.300592:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1787764106.300594:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764106.300596:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764106.300602:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00100000:1.0:1787764106.300603:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832691584:192.168.203.110@tcp:250:kworker.0
00000020:00000001:3.0:1787764106.300613:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:1.0:1787764106.300614:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:3.0:1787764106.300619:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:3.0:1787764106.300622:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:1.0:1787764106.300623:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:3.0:1787764106.300625:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
02000000:00000001:1.0:1787764106.300626:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764106.300632:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000001:3.0:1787764106.300636:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:3.0:1787764106.300640:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000010:1.0:1787764106.300644:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9000.
00000100:00000040:3.0:1787764106.300645:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764106.300652:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1787764106.300653:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764106.300663:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:1.0:1787764106.300673:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764106.300674:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764106.300683:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000400:00000010:1.0:1787764106.300684:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000100:00080000:3.0:1787764106.300692:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000010:1.0:1787764106.300694:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000001:3.0:1787764106.300696:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764106.300706:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:1.0:1787764106.300710:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832691584, portal 25
00000100:00000001:1.0:1787764106.300714:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764106.300716:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049849600 : -115096659702016 : ffff9751f87bcf00)
02000000:00000001:3.0:1787764106.300716:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764106.300731:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764106.300736:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bcf00 x1874599832691584/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764131 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764106.300748:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787764106.300759:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1787764106.300760:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764106.300768:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000008:00000001:3.0:1787764106.300770:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787764106.300772:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000001:3.0:1787764106.300774:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787764106.300779:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832691584, offset 0
00000100:00000010:3.0:1787764106.300782:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc780.
00000020:00000040:3.0:1787764106.300786:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000200:1.0:1787764106.300794:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
02000000:00000001:3.0:1787764106.300795:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764106.300800:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764106.300803:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764106.300804:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1787764106.300807:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:3.0:1787764106.300807:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764106.300819:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613400.
00000400:00000200:1.0:1787764106.300826:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:3.0:1787764106.300829:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764106.300835:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:3.0:1787764106.300843:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc780 x1874599832691712/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787764106.300844:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764106.300856:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764106.300867:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764106.300880:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58638:58638:256:4294967295] 192.168.203.110@tcp LPNI seq info [58638:58638:8:4294967295]
00000100:00000001:3.0:1787764106.300881:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764106.300895:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764106.300896:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764106.300915:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:3.0:1787764106.300915:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7d56184] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc780 x1874599832691712/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.0:1787764106.300930:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764106.300936:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764106.300952:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764106.300961:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000100:00000001:3.0:1787764106.300969:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787764106.300970:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00080000:3.0:1787764106.300976:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:1.1:1787764106.300978:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:1.0:1787764106.300988:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764106.300990:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000100:00080000:3.0:1787764106.300992:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1787764106.300995:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1787764106.300998:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:3.0:1787764106.300998:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764106.301005:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1787764106.301009:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:3.0:1787764106.301014:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21820
00000400:00000200:1.0:1787764106.301018:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764106.301024:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000400:00000200:3.0:1787764106.301024:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764106.301033:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000400:00000200:3.0:1787764106.301034:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764106.301042:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000200:1.0:1787764106.301045:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bcf00 x1874599832691584/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764131 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787764106.301055:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764106.301068:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764106.301079:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764106.301091:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1787764106.301091:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:1.0:1787764106.301096:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bcf00 x1874599832691584/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764131 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787764106.301100:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764106.301106:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000100:00000001:1.0:1787764106.301108:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764106.301112:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:3.0:1787764106.301112:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764106.301122:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000100:00000001:1.0:1787764106.301124:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787764106.301130:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:1.0:1787764106.301158:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764106.301160:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316494580
00000100:00000010:1.0:1787764106.301168:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4060.
00000020:00000001:3.0:1787764106.301172:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000001:1.0:1787764106.301178:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000040:3.0:1787764106.301181:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764106.301185:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764106.301190:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764106.301191:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316494580
00000100:00000001:3.0:1787764106.301201:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000010:1.0:1787764106.301202:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4b40.
00000020:00000040:3.0:1787764106.301215:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00000001:1.0:1787764106.301216:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787764106.301224:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:3.0:1787764106.301226:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764106.301228:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764106.301230:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764106.301232:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000001:3.0:1787764106.301235:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:1.0:1787764106.301241:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764106/real 1787764106]  req@ffff9751f87bcf00 x1874599832691584/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787764106.301242:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764106.301251:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764106.301262:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787764106.301269:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000001:1.0:1787764106.301272:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000008:00000001:3.0:1787764106.301274:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1787764106.301275:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:3.0:1787764106.301279:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787764106.301280:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764106.301290:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000100:00000010:3.0:1787764106.301290:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc3c0.
00000400:00000010:1.0:1787764106.301295:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764106.301301:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000040:3.0:1787764106.301303:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000200:1.0:1787764106.301309:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bcf00 x1874599832691584/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787764106.301311:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764106.301316:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764106.301328:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:1.0:1787764106.301330:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bcf00 x1874599832691584/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787764106.301334:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1787764106.301339:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764106.301342:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787764106.301342:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764106.301346:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764106.301348:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000010:3.0:1787764106.301350:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611000.
00000100:00100000:1.0:1787764106.301355:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bcf00 x1874599832691584/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764106.301357:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764106.301373:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764106.301374:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc3c0 x1874599832691840/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:1.0:1787764106.301385:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bcf00 x1874599832691584/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764106.301389:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764106.301395:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764106.301410:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000060bf8470] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc3c0 x1874599832691840/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.301411:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764106.301425:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764106.301434:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764106.301439:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764106.301441:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764106.301443:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000040:3.0:1787764106.301445:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787764106.301446:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:3.0:1787764106.301461:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787764106.301462:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764106.301473:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764106.301474:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764106.301484:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00080000:3.0:1787764106.301486:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787764106.301491:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000040:1.0:1787764106.301506:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bcf00 x1874599832691584/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787764106.301507:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764106.301518:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00100000:1.0:1787764106.301523:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bcf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832691584:192.168.203.110@tcp:250:kworker.0
00000100:00080000:3.0:1787764106.301527:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21820
00000100:00000001:1.0:1787764106.301530:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764106.301534:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764106.301542:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bcf00 x1874599832691584/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764106.301542:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764106.301554:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764106.301564:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787764106.301565:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764106.301569:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000040:3.0:1787764106.301571:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000010:1.0:1787764106.301574:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9000.
00000100:00000001:3.0:1787764106.301576:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1787764106.301584:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000040:3.0:1787764106.301585:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764106.301593:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764106.301594:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764106.301596:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764106.301602:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000001:1.0:1787764106.301603:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
00000020:00000040:3.0:1787764106.301607:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
02000000:00000010:1.0:1787764106.301609:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611400.
00000020:00000001:3.0:1787764106.301612:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:1.0:1787764106.301616:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000020:00000040:3.0:1787764106.301617:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
02000000:00000001:1.0:1787764106.301621:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000020:00000001:3.0:1787764106.301622:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000010:1.0:1787764106.301626:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bcf00.
00000020:00000001:3.0:1787764106.301628:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:1.0:1787764106.301631:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764106.301635:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764106.301636:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764106.301648:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764106.301648:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764106.301661:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764106.301663:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764106.301664:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787764106.301668:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1787764106.301669:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:1.0:1787764106.301672:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:3.0:1787764106.301677:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764106.301678:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764106.301684:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:3.0:1787764106.301684:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764106.301692:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:1.0:1787764106.301696:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc780 x1874599832691712/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787764106.301699:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764106.301709:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764106.301724:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764106.301725:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764106.301734:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:3.0:1787764106.301736:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
02000000:00000001:1.0:1787764106.301741:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00000001:3.0:1787764106.301743:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764106.301745:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764106.301746:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764106.301756:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764106.301758:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bfc00.
00000100:00100000:1.0:1787764106.301761:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832691712:192.168.203.110@tcp:38:kworker.0
00000020:00000040:3.0:1787764106.301763:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:1.0:1787764106.301772:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764106.301774:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764106.301775:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764106.301777:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764106.301778:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764106.301792:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764106.301793:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:3.0:1787764106.301803:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:1.0:1787764106.301805:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9000.
02000000:00000001:3.0:1787764106.301807:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764106.301808:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764106.301813:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610000.
00000400:00000200:1.0:1787764106.301814:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764106.301823:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787764106.301824:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764106.301832:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00100000:3.0:1787764106.301836:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bfc00 x1874599832691968/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:1.0:1787764106.301837:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832691712, portal 10
00000100:00000001:1.0:1787764106.301846:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764106.301850:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049847680 : -115096659703936 : ffff9751f87bc780)
00000100:00000001:3.0:1787764106.301860:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:1.0:1787764106.301864:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc780 x1874599832691712/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764131 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764106.301865:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764106.301879:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003c1f6b31] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bfc00 x1874599832691968/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.301889:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764106.301897:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764106.301901:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764106.301908:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832691712, offset 0
00000400:00000200:1.0:1787764106.301920:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764106.301927:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764106.301933:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:3.0:1787764106.301933:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764106.301943:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:3.0:1787764106.301946:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21831)
00000400:00000200:1.0:1787764106.301948:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764106.301955:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764106.301957:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764106.301960:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58639:58639:256:4294967295] 192.168.203.110@tcp LPNI seq info [58639:58639:8:4294967295]
00000400:00000200:1.0:1787764106.301964:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764106.301977:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764106.301981:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764106.301984:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764106.301987:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764106.301989:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787764106.301993:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764106.301995:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:1.0:1787764106.301998:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764106.301999:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764106.302001:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764106.302003:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764106.302005:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764106.302007:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764106.302009:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764106.302011:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc780 x1874599832691712/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764131 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302017:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764106.302019:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc780 x1874599832691712/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764131 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302023:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764106.302024:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764106.302026:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764106.302028:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787764106.302030:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764106.302031:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316493344
00000100:00000001:1.0:1787764106.302033:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316494580
00000100:00000001:1.0:1787764106.302035:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316493344
00000100:00000001:1.0:1787764106.302037:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316494580
00000100:00000001:1.0:1787764106.302039:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764106.302040:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764106.302042:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764106.302044:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764106/real 1787764106]  req@ffff9751f87bc780 x1874599832691712/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764106.302049:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764106.302050:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764106.302051:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764106.302052:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764106.302053:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764106.302054:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764106.302056:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc780 x1874599832691712/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764106.302061:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc780 x1874599832691712/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302066:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764106.302067:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764106.302068:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764106.302069:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764106.302071:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc780 x1874599832691712/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302075:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764106.302077:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc780 x1874599832691712/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302081:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764106.302083:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764106.302084:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764106.302084:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764106.302086:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764106.302096:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764106.302098:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764106.302101:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764106.302102:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764106.302105:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc780 x1874599832691712/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764106.302110:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832691712:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764106.302112:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764106.302114:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc3c0 x1874599832691840/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302119:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764106.302120:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764106.302121:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764106.302122:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764106.302123:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764106.302125:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832691840:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764106.302127:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764106.302129:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764106.302130:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764106.302131:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764106.302151:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764106.302153:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764106.302155:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764106.302156:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764106.302158:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764106.302160:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832691840, portal 10
00000100:00000001:1.0:1787764106.302162:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764106.302162:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049846720 : -115096659704896 : ffff9751f87bc3c0)
00000100:00000040:1.0:1787764106.302167:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc3c0 x1874599832691840/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764131 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302172:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764106.302173:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764106.302174:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764106.302176:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832691840, offset 0
00000400:00000200:1.0:1787764106.302178:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764106.302180:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764106.302182:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764106.302184:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764106.302186:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764106.302188:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764106.302189:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764106.302191:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58640:58640:256:4294967295] 192.168.203.110@tcp LPNI seq info [58640:58640:8:4294967295]
00000400:00000200:1.0:1787764106.302194:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764106.302198:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764106.302201:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764106.302203:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764106.302205:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764106.302207:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787764106.302209:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764106.302211:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:1.0:1787764106.302214:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764106.302215:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764106.302217:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764106.302219:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764106.302221:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764106.302222:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764106.302224:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764106.302226:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc3c0 x1874599832691840/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764131 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302231:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764106.302233:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc3c0 x1874599832691840/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764131 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302241:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764106.302242:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764106.302244:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764106.302246:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787764106.302248:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764106.302249:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316493333
00000100:00000001:1.0:1787764106.302251:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316494580
00000100:00000001:1.0:1787764106.302252:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316493333
00000100:00000001:1.0:1787764106.302254:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316494580
00000100:00000001:1.0:1787764106.302255:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764106.302256:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764106.302257:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764106.302259:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764106/real 1787764106]  req@ffff9751f87bc3c0 x1874599832691840/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764106.302264:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764106.302265:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764106.302266:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764106.302267:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764106.302268:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764106.302270:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764106.302271:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc3c0 x1874599832691840/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764106.302276:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc3c0 x1874599832691840/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302280:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764106.302281:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764106.302282:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764106.302283:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764106.302285:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc3c0 x1874599832691840/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302290:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764106.302292:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc3c0 x1874599832691840/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302297:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764106.302298:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764106.302299:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764106.302300:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764106.302301:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764106.302302:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764106.302303:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764106.302305:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764106.302306:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764106.302309:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc3c0 x1874599832691840/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764106.302313:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832691840:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764106.302316:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764106.302317:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764106.302319:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc780 x1874599832691712/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302323:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764106.302324:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764106.302326:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9000.
02000000:00000001:1.0:1787764106.302327:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764106.302328:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764106.302329:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764106.302330:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764106.302332:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613400.
02000000:00000001:1.0:1787764106.302334:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764106.302336:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764106.302337:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc780.
00000100:00000001:1.0:1787764106.302338:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764106.302339:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764106.302340:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764106.302342:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc3c0 x1874599832691840/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302346:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764106.302347:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764106.302348:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764106.302349:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764106.302350:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764106.302351:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764106.302352:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764106.302353:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611000.
02000000:00000001:1.0:1787764106.302354:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764106.302355:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764106.302355:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc3c0.
00000100:00000001:1.0:1787764106.302356:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764106.302357:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764106.302358:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764106.302360:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764106.302361:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764106.302363:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764106.302364:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764106.302364:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764106.302367:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bfc00 x1874599832691968/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302371:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764106.302372:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764106.302373:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764106.302374:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764106.302376:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764106.302377:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832691968:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764106.302381:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764106.302383:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764106.302384:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764106.302385:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764106.302387:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764106.302388:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764106.302390:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764106.302392:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764106.302394:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764106.302396:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832691968, portal 4
00000100:00000001:1.0:1787764106.302397:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764106.302398:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049861120 : -115096659690496 : ffff9751f87bfc00)
00000100:00000040:1.0:1787764106.302402:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bfc00 x1874599832691968/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764131 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302406:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764106.302408:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764106.302409:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764106.302411:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832691968, offset 0
00000400:00000200:1.0:1787764106.302413:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764106.302415:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764106.302417:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764106.302418:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764106.302420:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764106.302422:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764106.302423:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764106.302425:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58641:58641:256:4294967295] 192.168.203.110@tcp LPNI seq info [58641:58641:8:4294967295]
00000400:00000200:1.0:1787764106.302428:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764106.302432:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764106.302435:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764106.302436:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764106.302439:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764106.302441:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:1.1:1787764106.302444:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764106.302445:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:1.0:1787764106.302448:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764106.302449:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764106.302451:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764106.302453:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764106.302455:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764106.302456:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764106.302457:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764106.302460:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bfc00 x1874599832691968/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764131 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302464:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764106.302466:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bfc00 x1874599832691968/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764131 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302471:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764106.302472:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764106.302473:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764106.302475:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787764106.302477:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764106.302478:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316493350
00000100:00000001:1.0:1787764106.302479:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316494580
00000100:00000001:1.0:1787764106.302481:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316493350
00000100:00000001:1.0:1787764106.302482:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316494580
00000100:00000001:1.0:1787764106.302483:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764106.302485:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764106.302486:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764106.302488:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764106/real 1787764106]  req@ffff9751f87bfc00 x1874599832691968/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764106.302504:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764106.302505:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764106.302506:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764106.302508:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764106.302509:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764106.302511:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764106.302513:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bfc00 x1874599832691968/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764106.302518:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bfc00 x1874599832691968/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302522:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764106.302523:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764106.302524:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764106.302525:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764106.302527:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bfc00 x1874599832691968/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302531:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764106.302534:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bfc00 x1874599832691968/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302539:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764106.302540:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764106.302541:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764106.302542:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764106.302543:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764106.302544:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764106.302546:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764106.302548:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764106.302549:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764106.302551:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bfc00 x1874599832691968/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764106.302556:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832691968:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764106.302559:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764106.302560:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764106.302562:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bfc00 x1874599832691968/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764131 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764106.302567:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764106.302567:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764106.302569:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764106.302570:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764106.302571:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764106.302571:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787764106.302573:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764106.302573:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610000.
02000000:00000001:1.0:1787764106.302575:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764106.302576:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764106.302577:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bfc00.
00000100:00000001:1.0:1787764106.302579:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764106.302580:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764106.302581:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764106.302583:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764106.302583:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764106.302585:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764106.302586:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764106.940252:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764106.940261:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764106.940264:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764106.940265:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764106.940275:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764106.940276:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764106.940290:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764106.940293:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764106.940296:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764106.940296:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764106.940311:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764106.940313:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764106.940314:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764106.940316:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764106.940321:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764106.940378:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764106.940382:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764106.940385:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764107.324263:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764107.324284:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764107.324288:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764107.324294:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764107.324298:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764107.324302:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764107.324305:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764107.324307:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764107.324309:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764107.580254:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316494580, at 4316494592
00000100:00000010:3.0:1787764107.580283:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4b40.
00000100:00000040:2.0:1787764107.580377:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316494580, at 4316494592
00000100:00000010:2.0:1787764107.580396:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4060.
00000100:00000001:2.0:1787764107.964253:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764107.964254:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764107.964260:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764107.964261:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764107.964264:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764107.964267:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764107.964268:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764107.964271:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764107.964274:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764107.964276:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764107.964278:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764107.964280:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764107.964283:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764107.964284:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764107.964285:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764107.964289:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764107.964293:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764107.964296:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764107.964299:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764107.964303:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764107.964304:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764107.964365:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764107.964368:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764107.964371:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764108.348258:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764108.348266:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764108.348268:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764108.348273:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764108.348277:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764108.348282:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764108.348285:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764108.348287:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764108.348290:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764108.988193:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764108.988203:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764108.988206:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764108.988212:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764108.988213:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764108.988217:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764108.988221:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764108.988224:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764108.988227:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764108.988230:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764108.988232:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764108.988233:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764108.988240:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764108.988240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764108.988243:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764108.988297:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764108.988301:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764108.988304:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764109.372238:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764109.372244:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764109.372246:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764109.372251:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764109.372254:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764109.372258:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764109.372260:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764109.372262:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764109.372264:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764110.012261:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764110.012267:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764110.012270:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764110.012272:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764110.012276:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764110.012280:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764110.012284:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764110.012287:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764110.012289:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764110.012292:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764110.012297:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764110.012303:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764110.012306:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764110.012309:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764110.012313:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764110.012359:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764110.012365:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764110.012369:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764110.396211:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764110.396217:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764110.396219:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764110.396224:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764110.396234:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764110.396237:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764110.396240:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764110.396242:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764110.396245:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764111.036261:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764111.036271:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764111.036275:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764111.036278:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764111.036281:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764111.036285:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764111.036290:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764111.036293:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764111.036295:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764111.036297:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764111.036300:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764111.036303:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764111.036304:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764111.036309:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764111.036313:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764111.036400:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764111.036405:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764111.036408:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787764111.420284:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764111.420300:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764111.420304:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787764111.420309:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:3.0:1787764111.420309:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764111.420317:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764111.420320:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764111.420320:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00100000:2.0:1787764111.420330:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:3.0:1787764111.420337:0:304425:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21826
00000100:00000001:2.0:1787764111.420340:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764111.420346:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764111.420353:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000400:00000200:3.0:1787764111.420353:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764111.420359:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764111.420365:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764111.420367:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764111.420374:0:304425:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764111.420379:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764111.420383:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764111.420387:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764111.420390:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764111.420395:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764111.420398:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787764111.420401:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764111.420405:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787764111.420409:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787764111.420413:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787764111.420416:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764111.420419:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764111.420421:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764111.420424:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764111.420427:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787764111.420432:0:304425:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764111.420435:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764111.420439:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764111.420443:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764111.420447:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764111.420449:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764111.420453:0:304425:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764111.420480:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bde00.
00000020:00000040:3.0:1787764111.420484:0:304425:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787764111.420490:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764111.420492:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764111.420496:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764111.420500:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764111.420506:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764111.420517:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610000.
00000100:00000001:3.0:1787764111.420526:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764111.420537:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bde00 x1874599832692096/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764111.420549:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764111.420555:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764111.420560:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000941163c2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bde00 x1874599832692096/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764111.420585:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764111.420589:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764111.420594:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787764111.420597:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764111.420599:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764111.420604:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764111.420608:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21826
00000100:00000001:1.0:1787764111.420613:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:3.0:1787764111.420614:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764111.420620:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764111.420625:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000400:00000200:3.0:1787764111.420625:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764111.420632:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764111.420640:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bde00 x1874599832692096/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787764111.420642:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764111.420652:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764111.420663:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764111.420664:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764111.420666:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764111.420673:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:3.0:1787764111.420674:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:1.0:1787764111.420677:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:3.0:1787764111.420681:0:304425:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:1.0:1787764111.420682:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1787764111.420686:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00100000:1.0:1787764111.420689:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832692096:192.168.203.110@tcp:250:kworker.0
00000020:00000001:3.0:1787764111.420691:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:1.0:1787764111.420695:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:3.0:1787764111.420697:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
02000000:00000001:1.0:1787764111.420701:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764111.420704:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787764111.420704:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:1.0:1787764111.420709:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000001:3.0:1787764111.420714:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000010:1.0:1787764111.420719:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb400.
00000100:00000040:3.0:1787764111.420722:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:1.0:1787764111.420723:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764111.420730:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787764111.420734:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764111.420740:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000100:00000040:3.0:1787764111.420740:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764111.420749:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000010:1.0:1787764111.420753:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000020:00000040:3.0:1787764111.420755:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00000200:1.0:1787764111.420761:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832692096, portal 25
00000100:00080000:3.0:1787764111.420763:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764111.420765:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764111.420769:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049853440 : -115096659698176 : ffff9751f87bde00)
00000100:00000001:3.0:1787764111.420772:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764111.420782:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bde00 x1874599832692096/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764136 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787764111.420782:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787764111.420792:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:3.0:1787764111.420795:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1787764111.420796:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764111.420799:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
02000000:00000001:3.0:1787764111.420803:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787764111.420805:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832692096, offset 0
00000100:00000001:3.0:1787764111.420809:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:1.0:1787764111.420813:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000008:00100000:3.0:1787764111.420817:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:1.0:1787764111.420825:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000008:00000001:3.0:1787764111.420827:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764111.420833:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764111.420836:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000010:3.0:1787764111.420840:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bed00.
00000400:00000200:1.0:1787764111.420841:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:3.0:1787764111.420850:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000200:1.0:1787764111.420851:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764111.420860:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
02000000:00000001:3.0:1787764111.420860:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:1.0:1787764111.420868:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
02000000:00000001:3.0:1787764111.420868:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764111.420876:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58642:58642:256:4294967295] 192.168.203.110@tcp LPNI seq info [58642:58642:8:4294967295]
00000100:00000001:3.0:1787764111.420876:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764111.420885:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1787764111.420886:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:3.0:1787764111.420891:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764111.420899:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
02000000:00000010:3.0:1787764111.420901:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611000.
00000800:00000200:1.0:1787764111.420908:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787764111.420910:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1787764111.420914:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764111.420923:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00100000:3.0:1787764111.420924:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bed00 x1874599832692224/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:1.0:1787764111.420928:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787764111.420947:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764111.420955:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000100:00000001:3.0:1787764111.420963:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000100:1.0:1787764111.420965:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764111.420968:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000100:00000001:3.0:1787764111.420970:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764111.420975:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:3.0:1787764111.420982:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000079751c9a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bed00 x1874599832692224/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787764111.420985:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764111.420994:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764111.420998:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764111.421006:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:3.0:1787764111.421016:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787764111.421017:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bde00 x1874599832692096/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764136 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787764111.421028:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787764111.421031:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00080000:3.0:1787764111.421038:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000040:1.0:1787764111.421039:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bde00 x1874599832692096/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764136 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764111.421046:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1787764111.421051:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764111.421052:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764111.421059:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764111.421061:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764111.421069:0:304425:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21826
00000100:00000001:1.0:1787764111.421070:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764111.421090:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764111.421091:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764111.421098:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316499699
00000400:00000200:3.0:1787764111.421102:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764111.421108:0:304425:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000010:1.0:1787764111.421109:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4f60.
00000100:00000040:3.0:1787764111.421115:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764111.421118:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787764111.421120:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787764111.421126:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316499699
00000100:00000040:3.0:1787764111.421126:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000010:1.0:1787764111.421158:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c44e0.
00000100:00000001:1.0:1787764111.421163:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787764111.421166:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764111.421168:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787764111.421169:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764111.421176:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000020:00000001:3.0:1787764111.421177:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764111.421182:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000100:00100000:1.0:1787764111.421184:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764111/real 1787764111]  req@ffff9751f87bde00 x1874599832692096/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787764111.421188:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764111.421194:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000400:00000001:1.0:1787764111.421200:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000001:3.0:1787764111.421202:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000001:1.0:1787764111.421203:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764111.421207:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764111.421211:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000020:00000001:3.0:1787764111.421211:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000010:1.0:1787764111.421214:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764111.421221:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:3.0:1787764111.421223:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000200:1.0:1787764111.421225:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bde00 x1874599832692096/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764111.421231:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764111.421243:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000200:1.0:1787764111.421248:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bde00 x1874599832692096/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764111.421252:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764111.421260:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00000001:1.0:1787764111.421264:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764111.421267:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764111.421273:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00080000:3.0:1787764111.421274:0:304425:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764111.421275:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764111.421283:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764111.421285:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bde00 x1874599832692096/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787764111.421291:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787764111.421300:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787764111.421302:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1787764111.421307:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bde00 x1874599832692096/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787764111.421310:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764111.421321:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764111.421321:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764111.421327:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000008:00100000:3.0:1787764111.421328:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1787764111.421330:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764111.421333:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764111.421342:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000008:00000001:3.0:1787764111.421342:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764111.421346:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764111.421352:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764111.421353:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764111.421362:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764111.421365:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000010:3.0:1787764111.421365:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc000.
00000020:00000040:3.0:1787764111.421378:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000040:1.0:1787764111.421379:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bde00 x1874599832692096/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787764111.421390:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764111.421397:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764111.421398:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832692096:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764111.421407:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1787764111.421411:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787764111.421411:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764111.421416:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764111.421417:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000010:3.0:1787764111.421425:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613400.
00000100:00000040:1.0:1787764111.421426:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bde00 x1874599832692096/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764111.421433:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764111.421447:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc000 x1874599832692352/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764111.421482:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764111.421487:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764111.421492:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764111.421499:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764111.421504:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764111.421507:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787764111.421515:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764111.421518:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610000.
02000000:00000001:1.0:1787764111.421526:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764111.421529:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000001:3.0:1787764111.421529:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000010:1.0:1787764111.421531:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bde00.
00000100:00000001:1.0:1787764111.421537:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764111.421537:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764111.421541:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764111.421546:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764111.421552:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000040:3.0:1787764111.421552:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3fc1c6c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc000 x1874599832692352/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.421555:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764111.421561:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764111.421565:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764111.421567:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764111.421575:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bed00 x1874599832692224/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.421594:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764111.421595:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764111.421596:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764111.421609:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:3.0:1787764111.421610:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:1.0:1787764111.421614:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764111.421620:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764111.421626:0:304425:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00100000:1.0:1787764111.421628:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832692224:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764111.421636:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:3.0:1787764111.421638:0:304425:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:1.0:1787764111.421639:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764111.421643:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764111.421649:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764111.421652:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb400.
00000100:00080000:3.0:1787764111.421654:0:304425:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
02000000:00000001:1.0:1787764111.421656:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764111.421659:0:304425:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:1.0:1787764111.421663:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:3.0:1787764111.421664:0:304425:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:1.0:1787764111.421667:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000100:00000001:3.0:1787764111.421672:0:304425:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:1.0:1787764111.421673:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764111.421679:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832692224, portal 10
00000100:00080000:3.0:1787764111.421683:0:304425:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21826
00000100:00000001:1.0:1787764111.421684:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764111.421688:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049857280 : -115096659694336 : ffff9751f87bed00)
00000400:00000200:3.0:1787764111.421696:0:304425:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:1.0:1787764111.421700:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bed00 x1874599832692224/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764136 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764111.421711:0:304425:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1787764111.421716:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:3.0:1787764111.421719:0:304425:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764111.421725:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764111.421727:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000040:3.0:1787764111.421730:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000200:1.0:1787764111.421732:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832692224, offset 0
00000100:00000001:3.0:1787764111.421740:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787764111.421747:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000040:3.0:1787764111.421751:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:1.0:1787764111.421758:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764111.421761:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:1.0:1787764111.421766:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000020:00000001:3.0:1787764111.421770:0:304425:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:1.0:1787764111.421772:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:3.0:1787764111.421774:0:304425:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000400:00000200:1.0:1787764111.421777:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000020:00000001:3.0:1787764111.421783:0:304425:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:1.0:1787764111.421784:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764111.421789:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000020:00000040:3.0:1787764111.421790:0:304425:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000400:00000200:1.0:1787764111.421796:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58643:58643:256:4294967295] 192.168.203.110@tcp LPNI seq info [58643:58643:8:4294967295]
00000020:00000001:3.0:1787764111.421799:0:304425:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000200:1.0:1787764111.421806:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:3.0:1787764111.421809:0:304425:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000200:1.0:1787764111.421820:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:3.0:1787764111.421825:0:304425:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:1.0:1787764111.421830:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787764111.421839:0:304425:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000010:1.0:1787764111.421840:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000100:00000040:3.0:1787764111.421851:0:304425:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000200:1.0:1787764111.421852:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764111.421858:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000100:00000001:3.0:1787764111.421861:0:304425:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:1.1:1787764111.421866:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:3.0:1787764111.421871:0:304425:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000800:00000200:1.1:1787764111.421871:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000100:00080000:3.0:1787764111.421891:0:304425:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000800:00000100:1.0:1787764111.421892:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764111.421898:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000100:00000001:3.0:1787764111.421902:0:304425:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764111.421904:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:3.0:1787764111.421911:0:304425:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787764111.421914:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764111.421918:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764111.421919:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
02000000:00000001:3.0:1787764111.421919:0:304425:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764111.421927:0:257575:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:3.0:1787764111.421928:0:304425:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787764111.421932:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bed00 x1874599832692224/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764136 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764111.421934:0:304425:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787764111.421940:0:304425:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:1.0:1787764111.421943:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000008:00000001:3.0:1787764111.421950:0:304425:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764111.421951:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bed00 x1874599832692224/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764136 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764111.421962:0:304425:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764111.421966:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764111.421969:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764111.421972:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764111.421972:0:304425:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87be1c0.
00000100:00000040:1.0:1787764111.421984:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000020:00000040:3.0:1787764111.421985:0:304425:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000400:00000200:1.0:1787764111.421990:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764111.421997:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316498464
02000000:00000001:3.0:1787764111.421997:0:304425:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1787764111.422003:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316499699
02000000:00000001:3.0:1787764111.422003:0:304425:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764111.422009:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316498464
00000100:00000001:3.0:1787764111.422011:0:304425:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1787764111.422013:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316499699
02000000:00000001:3.0:1787764111.422016:0:304425:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1787764111.422018:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764111.422022:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764111.422023:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000001:3.0:1787764111.422023:0:304425:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764111.422030:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764111/real 1787764111]  req@ffff9751f87bed00 x1874599832692224/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1787764111.422032:0:304425:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611400.
00000400:00000001:1.0:1787764111.422047:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:3.0:1787764111.422047:0:304425:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:1.0:1787764111.422056:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764111.422060:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000100:00100000:3.0:1787764111.422062:0:304425:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87be1c0 x1874599832692480/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787764111.422066:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764111.422071:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764111.422078:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:3.0:1787764111.422085:0:304425:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:1.0:1787764111.422086:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bed00 x1874599832692224/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764111.422091:0:304425:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764111.422104:0:304425:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e2011c6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87be1c0 x1874599832692480/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764111.422121:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bed00 x1874599832692224/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764111.422123:0:304425:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764111.422157:0:304425:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21836)
00000100:00000001:1.0:1787764111.422173:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764111.422176:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764111.422178:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764111.422180:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764111.422185:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bed00 x1874599832692224/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.422192:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764111.422196:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bed00 x1874599832692224/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.422203:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764111.422206:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764111.422207:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764111.422208:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764111.422211:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764111.422234:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764111.422238:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764111.422242:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764111.422245:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764111.422253:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bed00 x1874599832692224/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764111.422262:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832692224:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764111.422266:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764111.422268:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764111.422272:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bed00 x1874599832692224/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.422279:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764111.422281:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764111.422284:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764111.422288:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764111.422290:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764111.422291:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764111.422294:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764111.422296:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611000.
02000000:00000001:1.0:1787764111.422298:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764111.422300:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764111.422302:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bed00.
00000100:00000001:1.0:1787764111.422304:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764111.422305:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764111.422306:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764111.422309:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764111.422310:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764111.422312:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764111.422313:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764111.422314:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764111.422317:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc000 x1874599832692352/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.422323:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764111.422324:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764111.422326:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764111.422327:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764111.422328:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764111.422330:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832692352:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764111.422333:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764111.422335:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764111.422336:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764111.422337:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764111.422340:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764111.422341:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764111.422344:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764111.422346:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764111.422348:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764111.422350:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832692352, portal 10
00000100:00000001:1.0:1787764111.422353:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764111.422354:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049845760 : -115096659705856 : ffff9751f87bc000)
00000100:00000040:1.0:1787764111.422358:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc000 x1874599832692352/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764136 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.422362:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764111.422364:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764111.422365:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764111.422367:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832692352, offset 0
00000400:00000200:1.0:1787764111.422369:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764111.422372:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764111.422375:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764111.422377:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764111.422379:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764111.422381:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764111.422383:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764111.422385:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58644:58644:256:4294967295] 192.168.203.110@tcp LPNI seq info [58644:58644:8:4294967295]
00000400:00000200:1.0:1787764111.422388:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764111.422392:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764111.422395:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764111.422398:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764111.422402:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764111.422405:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787764111.422409:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764111.422412:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:1.0:1787764111.422416:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764111.422418:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764111.422421:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764111.422424:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764111.422428:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764111.422430:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764111.422432:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764111.422436:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc000 x1874599832692352/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764136 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.422452:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764111.422455:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc000 x1874599832692352/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764136 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.422473:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764111.422474:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764111.422477:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764111.422480:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787764111.422482:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764111.422484:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316498453
00000100:00000001:1.0:1787764111.422486:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316499699
00000100:00000001:1.0:1787764111.422488:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316498453
00000100:00000001:1.0:1787764111.422489:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316499699
00000100:00000001:1.0:1787764111.422491:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764111.422492:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764111.422493:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764111.422496:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764111/real 1787764111]  req@ffff9751f87bc000 x1874599832692352/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764111.422501:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764111.422502:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764111.422503:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764111.422505:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764111.422506:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764111.422507:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764111.422509:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc000 x1874599832692352/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764111.422515:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc000 x1874599832692352/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.422519:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764111.422521:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764111.422522:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764111.422523:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764111.422525:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc000 x1874599832692352/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.422529:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764111.422532:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc000 x1874599832692352/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.422536:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764111.422537:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764111.422538:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764111.422539:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764111.422540:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764111.422541:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764111.422543:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764111.422545:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764111.422546:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764111.422548:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc000 x1874599832692352/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764111.422553:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832692352:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764111.422555:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764111.422558:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87be1c0 x1874599832692480/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.422563:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764111.422564:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764111.422565:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764111.422566:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764111.422568:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764111.422570:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832692480:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764111.422572:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764111.422574:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764111.422575:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764111.422577:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764111.422579:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9000.
02000000:00000001:1.0:1787764111.422581:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764111.422583:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764111.422584:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764111.422586:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764111.422589:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832692480, portal 4
00000100:00000001:1.0:1787764111.422590:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764111.422591:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049854400 : -115096659697216 : ffff9751f87be1c0)
00000100:00000040:1.0:1787764111.422595:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87be1c0 x1874599832692480/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764136 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.422599:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764111.422600:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764111.422602:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764111.422603:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832692480, offset 0
00000400:00000200:1.0:1787764111.422605:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764111.422608:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764111.422610:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764111.422612:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764111.422614:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764111.422615:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764111.422617:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764111.422619:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58645:58645:256:4294967295] 192.168.203.110@tcp LPNI seq info [58645:58645:8:4294967295]
00000400:00000200:1.0:1787764111.422622:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764111.422626:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764111.422629:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764111.422631:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764111.422634:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764111.422636:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:1.1:1787764111.422639:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764111.422640:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:1.0:1787764111.422643:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764111.422644:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764111.422646:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764111.422649:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764111.422650:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764111.422652:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764111.422653:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764111.422655:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87be1c0 x1874599832692480/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764136 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.422660:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764111.422662:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87be1c0 x1874599832692480/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764136 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.422666:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764111.422667:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764111.422669:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764111.422671:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787764111.422673:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764111.422674:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316498470
00000100:00000001:1.0:1787764111.422676:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316499699
00000100:00000001:1.0:1787764111.422677:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316498470
00000100:00000001:1.0:1787764111.422679:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316499699
00000100:00000001:1.0:1787764111.422680:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764111.422681:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764111.422682:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764111.422684:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764111/real 1787764111]  req@ffff9751f87be1c0 x1874599832692480/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764111.422689:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764111.422690:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764111.422691:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764111.422692:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764111.422693:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764111.422695:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764111.422697:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87be1c0 x1874599832692480/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764111.422701:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87be1c0 x1874599832692480/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.422705:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764111.422706:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764111.422707:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764111.422708:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764111.422710:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87be1c0 x1874599832692480/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.422714:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764111.422716:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87be1c0 x1874599832692480/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.422720:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764111.422721:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764111.422722:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764111.422723:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764111.422724:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764111.422725:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764111.422727:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764111.422729:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764111.422730:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764111.422732:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87be1c0 x1874599832692480/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764111.422742:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832692480:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764111.422744:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764111.422745:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764111.422747:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc000 x1874599832692352/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.422751:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764111.422752:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764111.422753:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764111.422755:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764111.422755:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764111.422756:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764111.422757:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764111.422759:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613400.
02000000:00000001:1.0:1787764111.422760:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764111.422761:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764111.422762:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc000.
00000100:00000001:1.0:1787764111.422764:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764111.422765:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764111.422766:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764111.422768:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87be1c0 x1874599832692480/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764136 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764111.422772:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764111.422772:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764111.422773:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9000.
02000000:00000001:1.0:1787764111.422774:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764111.422775:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764111.422776:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787764111.422777:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764111.422778:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611400.
02000000:00000001:1.0:1787764111.422779:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764111.422779:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764111.422780:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87be1c0.
00000100:00000001:1.0:1787764111.422782:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764111.422783:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764111.422784:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764111.422787:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764111.422787:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764111.422789:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764111.422790:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764112.060262:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764112.060273:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764112.060275:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764112.060279:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764112.060280:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764112.060287:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764112.060288:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764112.060294:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764112.060298:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764112.060298:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764112.060301:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764112.060303:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764112.060305:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764112.060316:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764112.060324:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764112.060327:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764112.060331:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764112.060336:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764112.060338:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764112.060341:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764112.060344:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764112.060400:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764112.060404:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764112.060407:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764112.444243:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764112.444252:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764112.444255:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764112.444262:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764112.444267:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764112.444271:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764112.444274:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764112.444277:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764112.444281:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764112.700238:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316499699, at 4316499712
00000100:00000010:3.0:1787764112.700259:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c44e0.
00000100:00000040:2.0:1787764112.700279:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316499699, at 4316499712
00000100:00000010:2.0:1787764112.700292:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4f60.
00000100:00000001:2.0:1787764113.084211:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764113.084217:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764113.084219:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764113.084224:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764113.084228:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764113.084232:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764113.084235:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764113.084238:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764113.084242:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764113.084253:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764113.084258:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764113.084263:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764113.084265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764113.084268:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764113.084269:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764113.084332:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764113.084336:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764113.084340:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764113.468285:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764113.468293:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764113.468295:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764113.468300:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764113.468305:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764113.468310:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764113.468313:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764113.468315:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764113.468318:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764114.108207:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764114.108214:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764114.108216:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764114.108221:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764114.108224:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764114.108227:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764114.108230:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764114.108231:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764114.108232:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764114.108237:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764114.108238:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764114.108247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764114.108250:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764114.108255:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764114.108257:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764114.108311:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764114.108315:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764114.108318:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764114.492214:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764114.492223:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764114.492226:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764114.492232:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764114.492237:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764114.492241:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764114.492244:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764114.492246:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764114.492249:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764115.132242:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764115.132247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764115.132251:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764115.132252:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764115.132253:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764115.132255:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764115.132258:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764115.132259:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764115.132261:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764115.132263:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764115.132264:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764115.132266:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764115.132266:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764115.132269:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764115.132269:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764115.132272:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764115.132273:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764115.132275:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764115.132275:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764115.132278:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764115.132280:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764115.132341:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764115.132345:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764115.132347:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764115.516255:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764115.516265:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764115.516267:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764115.516273:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764115.516278:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764115.516282:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764115.516285:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764115.516288:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764115.516292:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764116.156230:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764116.156234:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764116.156239:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764116.156240:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764116.156244:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764116.156248:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764116.156251:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764116.156251:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764116.156258:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764116.156259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764116.156263:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764116.156263:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764116.156268:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764116.156271:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764116.156274:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764116.156320:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764116.156324:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764116.156328:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764116.540201:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764116.540210:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764116.540213:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764116.540219:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:0.0:1787764116.540221:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764116.540224:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764116.540230:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787764116.540231:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787764116.540237:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787764116.540237:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764116.540242:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787764116.540243:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764116.540249:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764116.540250:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764116.540260:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21831
00000400:00000200:0.0:1787764116.540269:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764116.540276:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764116.540280:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764116.540283:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764116.540287:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764116.540290:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764116.540292:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764116.540297:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764116.540299:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764116.540302:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764116.540305:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764116.540308:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764116.540311:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787764116.540314:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764116.540316:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764116.540319:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764116.540321:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764116.540324:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787764116.540328:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764116.540331:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764116.540333:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764116.540337:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764116.540340:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764116.540343:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764116.540346:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764116.540354:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000020:00000040:0.0:1787764116.540358:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764116.540361:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764116.540363:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764116.540366:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764116.540369:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764116.540373:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764116.540382:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787764116.540389:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764116.540397:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832692608/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764116.540415:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764116.540420:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764116.540440:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832692608/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764116.540459:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764116.540462:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764116.540466:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764116.540468:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764116.540470:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764116.540474:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764116.540477:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21831
00000400:00000200:0.0:1787764116.540482:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764116.540483:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:0.0:1787764116.540487:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1787764116.540487:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764116.540491:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00080000:0.0:1787764116.540492:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764116.540500:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000040:1.0:1787764116.540501:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cb40 x1874599832692608/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764116.540504:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764116.540510:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764116.540516:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764116.540517:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787764116.540518:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764116.540523:0:304484:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:1.0:1787764116.540524:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:0.0:1787764116.540527:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
02000000:00000001:1.0:1787764116.540528:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:0.0:1787764116.540532:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:1.0:1787764116.540533:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787764116.540538:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00100000:1.0:1787764116.540539:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832692608:192.168.203.110@tcp:250:kworker.0
00000020:00000001:0.0:1787764116.540544:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:1.0:1787764116.540545:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:0.0:1787764116.540550:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:1.0:1787764116.540550:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764116.540553:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764116.540556:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:0.0:1787764116.540558:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000010:1.0:1787764116.540562:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9000.
00000100:00000001:0.0:1787764116.540563:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1787764116.540566:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764116.540570:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:1.0:1787764116.540575:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764116.540577:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000010:1.0:1787764116.540579:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000020:00000040:0.0:1787764116.540584:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000400:00000010:1.0:1787764116.540588:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00080000:0.0:1787764116.540591:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000200:1.0:1787764116.540595:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832692608, portal 25
00000100:00000001:0.0:1787764116.540596:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764116.540600:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:0.0:1787764116.540602:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787764116.540603:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049520960 : -115096660030656 : ffff9751f876cb40)
02000000:00000001:0.0:1787764116.540609:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1787764116.540610:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cb40 x1874599832692608/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764141 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787764116.540616:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764116.540622:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764116.540628:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1787764116.540629:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764116.540634:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000008:00000001:0.0:1787764116.540635:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787764116.540637:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000001:0.0:1787764116.540639:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787764116.540641:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832692608, offset 0
00000100:00000010:0.0:1787764116.540645:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
00000020:00000040:0.0:1787764116.540651:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000200:1.0:1787764116.540651:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
02000000:00000001:0.0:1787764116.540656:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764116.540658:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764116.540660:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764116.540661:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1787764116.540663:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787764116.540665:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764116.540671:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000400:00000200:1.0:1787764116.540675:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:0.0:1787764116.540677:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764116.540684:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:0.0:1787764116.540687:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832692736/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787764116.540693:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764116.540700:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1787764116.540703:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764116.540707:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764116.540707:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764116.540713:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58646:58646:256:4294967295] 192.168.203.110@tcp LPNI seq info [58646:58646:8:4294967295]
00000100:00000040:0.0:1787764116.540717:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832692736/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787764116.540720:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764116.540733:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764116.540734:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00080000:0.0:1787764116.540739:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:1.0:1787764116.540744:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00080000:0.0:1787764116.540747:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787764116.540750:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000010:1.0:1787764116.540750:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000100:00080000:0.0:1787764116.540752:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764116.540756:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:1.0:1787764116.540758:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00080000:0.0:1787764116.540762:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21831
00000800:00000200:1.0:1787764116.540762:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787764116.540771:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000400:00000200:0.0:1787764116.540772:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:1.1:1787764116.540778:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000400:00000200:0.0:1787764116.540781:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000800:00000100:1.0:1787764116.540786:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00080000:0.0:1787764116.540788:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000010:1.0:1787764116.540789:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000100:00000040:0.0:1787764116.540795:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:1.0:1787764116.540796:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1787764116.540801:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787764116.540804:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:0.0:1787764116.540808:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:1.0:1787764116.540811:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000100:00000001:0.0:1787764116.540812:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000010:1.0:1787764116.540814:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764116.540819:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764116.540822:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cb40 x1874599832692608/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764141 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787764116.540825:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764116.540829:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787764116.540832:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764116.540836:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000100:00000001:1.0:1787764116.540836:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000001:0.0:1787764116.540840:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787764116.540841:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cb40 x1874599832692608/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764141 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787764116.540846:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787764116.540854:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764116.540859:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764116.540864:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764116.540866:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764116.540868:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787764116.540874:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764116.540877:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764116.540878:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764116.540879:0:257575:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:0.0:1787764116.540884:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764116.540888:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764116.540888:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764116.540894:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764116.540894:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764116.540899:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764116.540900:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316504819
00000008:00100000:0.0:1787764116.540905:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000010:1.0:1787764116.540906:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c49c0.
00000008:00000001:0.0:1787764116.540910:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764116.540914:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764116.540915:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000010:0.0:1787764116.540919:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000100:00000001:1.0:1787764116.540922:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316504819
00000020:00000040:0.0:1787764116.540924:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000010:1.0:1787764116.540927:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4f60.
02000000:00000001:0.0:1787764116.540929:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1787764116.540932:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
02000000:00000001:0.0:1787764116.540933:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764116.540938:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1787764116.540940:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764116.540941:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764116.540945:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764116.540946:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764116.540951:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000010:0.0:1787764116.540952:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00100000:1.0:1787764116.540958:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764116/real 1787764116]  req@ffff9751f876cb40 x1874599832692608/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764116.540959:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764116.540969:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832692864/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000001:1.0:1787764116.540978:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:0.0:1787764116.540981:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000001:1.0:1787764116.540982:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764116.540986:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000100:00000001:0.0:1787764116.540987:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764116.540989:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764116.540992:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764116.540996:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:0.0:1787764116.540997:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832692864/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764116.541002:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cb40 x1874599832692608/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764116.541029:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787764116.541030:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cb40 x1874599832692608/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764116.541035:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787764116.541039:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000040:0.0:1787764116.541041:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:1.0:1787764116.541042:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764116.541046:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764116.541047:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764116.541050:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cb40 x1874599832692608/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787764116.541052:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764116.541060:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787764116.541063:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764116.541065:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764116.541065:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764116.541070:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:1.0:1787764116.541073:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cb40 x1874599832692608/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787764116.541075:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21831
00000400:00000200:0.0:1787764116.541084:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764116.541088:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000400:00000200:0.0:1787764116.541090:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1787764116.541093:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787764116.541094:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787764116.541097:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:0.0:1787764116.541099:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764116.541100:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764116.541104:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787764116.541105:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764116.541108:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:0.0:1787764116.541109:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00080000:1.0:1787764116.541111:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:0.0:1787764116.541112:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764116.541115:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000100:00080000:1.0:1787764116.541117:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000020:00000040:0.0:1787764116.541118:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787764116.541122:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:1.0:1787764116.541123:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000020:00000040:0.0:1787764116.541125:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787764116.541129:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:1.0:1787764116.541130:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cb40 x1874599832692608/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787764116.541132:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787764116.541161:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764116.541164:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:1.0:1787764116.541166:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832692608:192.168.203.110@tcp:250:kworker.0
00000100:00000040:0.0:1787764116.541168:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764116.541173:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787764116.541174:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:0.0:1787764116.541175:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:1.0:1787764116.541178:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00080000:0.0:1787764116.541180:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000040:1.0:1787764116.541184:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cb40 x1874599832692608/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764116.541185:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764116.541190:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764116.541194:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764116.541195:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:0.0:1787764116.541197:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764116.541198:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000001:0.0:1787764116.541200:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000010:1.0:1787764116.541201:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9000.
00000008:00100000:0.0:1787764116.541203:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
02000000:00000001:1.0:1787764116.541205:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000008:00000001:0.0:1787764116.541208:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764116.541209:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764116.541212:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000100:00000001:0.0:1787764116.541213:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764116.541217:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
00000100:00000010:0.0:1787764116.541219:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
02000000:00000010:1.0:1787764116.541220:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
00000020:00000040:0.0:1787764116.541223:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787764116.541226:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764116.541227:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764116.541228:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764116.541230:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764116.541230:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:0.0:1787764116.541231:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764116.541234:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
02000000:00000010:0.0:1787764116.541237:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000010:1.0:1787764116.541237:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cb40.
00000100:00000001:0.0:1787764116.541243:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764116.541244:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764116.541247:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764116.541250:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:0.0:1787764116.541252:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832692992/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764116.541257:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764116.541260:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764116.541261:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764116.541262:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787764116.541265:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764116.541266:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764116.541269:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787764116.541273:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832692992/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787764116.541276:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876da40 x1874599832692736/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764116.541294:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764116.541295:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764116.541298:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764116.541300:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:0.0:1787764116.541301:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21841)
02000000:00000001:1.0:1787764116.541302:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764116.541306:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764116.541308:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832692736:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764116.541311:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764116.541312:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764116.541314:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764116.541315:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764116.541318:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9000.
02000000:00000001:1.0:1787764116.541320:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764116.541323:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764116.541325:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764116.541328:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764116.541330:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832692736, portal 10
00000100:00000001:1.0:1787764116.541331:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764116.541332:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049524800 : -115096660026816 : ffff9751f876da40)
00000100:00000040:1.0:1787764116.541337:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876da40 x1874599832692736/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764141 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541341:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764116.541343:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764116.541345:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764116.541346:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832692736, offset 0
00000400:00000200:1.0:1787764116.541349:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764116.541352:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764116.541355:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764116.541357:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764116.541359:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764116.541361:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764116.541363:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764116.541365:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58647:58647:256:4294967295] 192.168.203.110@tcp LPNI seq info [58647:58647:8:4294967295]
00000400:00000200:1.0:1787764116.541368:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764116.541372:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764116.541376:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764116.541378:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764116.541380:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764116.541382:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787764116.541386:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764116.541387:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1787764116.541390:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764116.541391:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764116.541393:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764116.541396:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764116.541398:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764116.541399:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764116.541400:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764116.541403:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876da40 x1874599832692736/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764141 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541408:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764116.541410:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876da40 x1874599832692736/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764141 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541415:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764116.541416:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764116.541418:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764116.541435:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787764116.541439:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764116.541441:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316503584
00000100:00000001:1.0:1787764116.541443:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316504819
00000100:00000001:1.0:1787764116.541445:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316503584
00000100:00000001:1.0:1787764116.541447:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316504819
00000100:00000001:1.0:1787764116.541448:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764116.541450:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764116.541451:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764116.541454:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764116/real 1787764116]  req@ffff9751f876da40 x1874599832692736/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764116.541459:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764116.541460:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764116.541461:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764116.541463:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764116.541464:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764116.541465:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764116.541467:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876da40 x1874599832692736/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764116.541473:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876da40 x1874599832692736/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541477:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764116.541478:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764116.541479:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764116.541480:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764116.541482:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876da40 x1874599832692736/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541486:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764116.541489:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876da40 x1874599832692736/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541494:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764116.541495:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764116.541496:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764116.541497:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764116.541499:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764116.541507:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764116.541509:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764116.541511:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764116.541512:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764116.541515:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876da40 x1874599832692736/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764116.541520:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832692736:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764116.541523:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764116.541525:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876de00 x1874599832692864/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541529:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764116.541529:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764116.541531:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764116.541532:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764116.541533:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764116.541535:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832692864:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764116.541537:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764116.541538:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764116.541539:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764116.541540:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764116.541543:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764116.541544:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764116.541547:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764116.541548:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764116.541550:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764116.541552:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832692864, portal 10
00000100:00000001:1.0:1787764116.541554:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764116.541554:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049525760 : -115096660025856 : ffff9751f876de00)
00000100:00000040:1.0:1787764116.541558:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876de00 x1874599832692864/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764141 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541563:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764116.541564:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764116.541565:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764116.541567:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832692864, offset 0
00000400:00000200:1.0:1787764116.541569:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764116.541572:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764116.541574:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764116.541576:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764116.541578:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764116.541580:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764116.541581:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764116.541583:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58648:58648:256:4294967295] 192.168.203.110@tcp LPNI seq info [58648:58648:8:4294967295]
00000400:00000200:1.0:1787764116.541586:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764116.541590:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764116.541593:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764116.541595:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764116.541598:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764116.541599:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787764116.541602:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764116.541604:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1787764116.541607:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764116.541608:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764116.541610:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764116.541612:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764116.541614:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764116.541616:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764116.541617:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764116.541619:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876de00 x1874599832692864/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764141 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541624:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764116.541626:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876de00 x1874599832692864/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764141 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541630:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764116.541631:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764116.541633:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764116.541635:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787764116.541637:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764116.541638:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316503572
00000100:00000001:1.0:1787764116.541639:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316504819
00000100:00000001:1.0:1787764116.541640:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316503572
00000100:00000001:1.0:1787764116.541642:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316504819
00000100:00000001:1.0:1787764116.541644:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764116.541645:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764116.541646:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764116.541648:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764116/real 1787764116]  req@ffff9751f876de00 x1874599832692864/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764116.541655:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764116.541656:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764116.541658:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764116.541660:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764116.541662:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764116.541664:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764116.541667:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876de00 x1874599832692864/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764116.541676:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876de00 x1874599832692864/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541684:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764116.541685:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764116.541688:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764116.541689:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764116.541694:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876de00 x1874599832692864/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541701:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764116.541705:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876de00 x1874599832692864/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541710:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764116.541712:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764116.541713:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764116.541714:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764116.541715:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764116.541716:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764116.541717:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764116.541719:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764116.541721:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764116.541723:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876de00 x1874599832692864/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764116.541728:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832692864:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764116.541731:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764116.541733:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764116.541735:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876da40 x1874599832692736/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541739:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764116.541740:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764116.541742:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9000.
02000000:00000001:1.0:1787764116.541744:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764116.541745:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764116.541746:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764116.541747:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764116.541748:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:1.0:1787764116.541750:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764116.541751:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764116.541752:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876da40.
00000100:00000001:1.0:1787764116.541754:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764116.541755:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764116.541756:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764116.541758:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876de00 x1874599832692864/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541761:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764116.541762:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764116.541763:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764116.541764:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764116.541765:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764116.541766:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764116.541767:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764116.541768:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017800.
02000000:00000001:1.0:1787764116.541770:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764116.541770:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764116.541771:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876de00.
00000100:00000001:1.0:1787764116.541774:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764116.541774:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764116.541776:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764116.541778:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764116.541779:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764116.541780:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764116.541781:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764116.541782:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764116.541785:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832692992/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541789:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764116.541790:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764116.541792:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764116.541793:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764116.541794:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764116.541796:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832692992:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764116.541798:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764116.541799:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764116.541800:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764116.541801:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764116.541803:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764116.541805:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764116.541808:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764116.541809:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764116.541812:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764116.541814:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832692992, portal 4
00000100:00000001:1.0:1787764116.541815:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764116.541816:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
00000100:00000040:1.0:1787764116.541820:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832692992/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764141 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541824:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764116.541825:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764116.541827:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764116.541829:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832692992, offset 0
00000400:00000200:1.0:1787764116.541831:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764116.541833:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764116.541836:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764116.541838:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764116.541839:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764116.541841:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764116.541843:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764116.541845:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58649:58649:256:4294967295] 192.168.203.110@tcp LPNI seq info [58649:58649:8:4294967295]
00000400:00000200:1.0:1787764116.541847:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764116.541852:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764116.541855:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764116.541857:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764116.541860:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764116.541861:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:1.1:1787764116.541864:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764116.541866:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:1.0:1787764116.541869:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764116.541871:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764116.541873:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764116.541875:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764116.541877:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764116.541878:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764116.541880:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764116.541882:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876c000 x1874599832692992/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764141 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541887:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764116.541889:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832692992/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764141 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541893:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764116.541894:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764116.541896:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764116.541898:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787764116.541899:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764116.541901:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316503589
00000100:00000001:1.0:1787764116.541903:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316504819
00000100:00000001:1.0:1787764116.541904:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316503589
00000100:00000001:1.0:1787764116.541906:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316504819
00000100:00000001:1.0:1787764116.541907:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764116.541908:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764116.541909:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764116.541911:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764116/real 1787764116]  req@ffff9751f876c000 x1874599832692992/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764116.541917:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764116.541918:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764116.541919:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764116.541920:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764116.541921:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764116.541923:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764116.541924:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832692992/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764116.541930:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832692992/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541942:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764116.541944:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764116.541946:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764116.541947:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764116.541951:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832692992/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541959:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764116.541963:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832692992/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.541971:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764116.541974:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764116.541975:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764116.541977:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764116.541979:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764116.541981:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764116.541983:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764116.541985:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764116.541987:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764116.541990:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832692992/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764116.541995:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832692992:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764116.541998:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764116.541999:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764116.542001:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832692992/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764141 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764116.542005:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764116.542006:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764116.542007:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764116.542009:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764116.542010:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764116.542011:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787764116.542012:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764116.542013:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016c00.
02000000:00000001:1.0:1787764116.542015:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764116.542016:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764116.542017:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:1.0:1787764116.542018:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764116.542019:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764116.542020:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764116.542022:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764116.542023:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764116.542025:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764116.542026:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764117.180216:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764117.180216:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764117.180223:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764117.180226:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764117.180230:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764117.180231:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764117.180235:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764117.180237:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764117.180240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764117.180244:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764117.180244:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764117.180249:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764117.180253:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764117.180255:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764117.180257:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764117.180296:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764117.180300:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764117.180303:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764117.564225:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764117.564231:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764117.564233:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764117.564237:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764117.564240:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764117.564243:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764117.564245:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764117.564247:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764117.564249:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764117.820260:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316504819, at 4316504832
00000100:00000010:3.0:1787764117.820277:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4f60.
00000100:00000040:2.0:1787764117.820296:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316504819, at 4316504832
00000100:00000010:2.0:1787764117.820306:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c49c0.
00000100:00000001:2.0:1787764118.204234:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764118.204238:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764118.204242:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764118.204244:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764118.204247:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764118.204253:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764118.204253:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764118.204259:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764118.204260:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764118.204263:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764118.204265:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764118.204267:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764118.204271:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764118.204272:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764118.204275:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764118.204276:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764118.204280:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764118.204281:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764118.204287:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764118.204291:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764118.204293:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764118.204353:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764118.204356:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764118.204359:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764118.588255:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764118.588263:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764118.588266:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764118.588272:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764118.588276:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764118.588280:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764118.588283:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764118.588285:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764118.588288:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764119.228211:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764119.228217:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764119.228219:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764119.228225:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764119.228228:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764119.228229:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764119.228233:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764119.228236:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764119.228240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764119.228242:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764119.228244:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764119.228247:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764119.228251:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764119.228254:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764119.228258:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764119.228289:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764119.228294:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764119.228298:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764119.612260:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764119.612269:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764119.612272:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764119.612278:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764119.612282:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764119.612286:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764119.612289:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764119.612292:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764119.612295:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764120.252214:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764120.252217:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764120.252221:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764120.252221:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764120.252226:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764120.252228:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764120.252230:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764120.252231:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764120.252237:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764120.252239:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764120.252241:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764120.252243:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764120.252245:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764120.252248:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764120.252251:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764120.252299:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764120.252302:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764120.252305:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764120.636232:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764120.636239:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764120.636242:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764120.636248:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764120.636252:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764120.636264:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764120.636268:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764120.636270:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764120.636273:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764121.276240:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764121.276248:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764121.276252:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764121.276252:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764121.276258:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764121.276260:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764121.276266:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764121.276266:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764121.276270:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764121.276272:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764121.276276:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764121.276277:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764121.276279:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764121.276282:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764121.276287:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764121.276342:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764121.276347:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764121.276351:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787764121.660253:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764121.660253:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764121.660260:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1787764121.660261:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787764121.660264:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764121.660265:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:2.0:1787764121.660269:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:0.0:1787764121.660270:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764121.660277:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787764121.660277:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764121.660284:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787764121.660288:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21836
00000100:00000001:2.0:1787764121.660289:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764121.660293:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764121.660295:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787764121.660300:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764121.660308:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764121.660311:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764121.660314:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764121.660317:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764121.660321:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764121.660323:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764121.660327:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764121.660328:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764121.660330:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764121.660334:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764121.660336:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764121.660338:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787764121.660340:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764121.660341:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764121.660342:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764121.660343:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764121.660345:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787764121.660347:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764121.660349:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764121.660351:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764121.660354:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764121.660356:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764121.660357:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764121.660359:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764121.660367:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787764121.660369:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764121.660372:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764121.660374:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764121.660376:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764121.660378:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764121.660381:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764121.660402:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787764121.660408:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764121.660417:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832693120/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764121.660428:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764121.660432:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764121.660437:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832693120/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764121.660460:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764121.660463:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764121.660467:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764121.660469:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764121.660470:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764121.660474:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764121.660486:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21836
00000100:00000001:1.0:1787764121.660488:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:0.0:1787764121.660493:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764121.660495:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000400:00000200:0.0:1787764121.660500:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1787764121.660500:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00080000:0.0:1787764121.660505:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764121.660512:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832693120/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764121.660513:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764121.660518:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764121.660522:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764121.660524:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764121.660526:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764121.660528:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787764121.660531:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764121.660535:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787764121.660539:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787764121.660544:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787764121.660551:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764121.660557:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764121.660562:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764121.660566:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787764121.660566:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764121.660570:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787764121.660572:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:1.0:1787764121.660575:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:0.0:1787764121.660579:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:1.0:1787764121.660581:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764121.660584:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764121.660587:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764121.660589:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:1.0:1787764121.660595:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832693120:192.168.203.110@tcp:250:kworker.0
02000000:00000001:0.0:1787764121.660598:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764121.660601:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787764121.660603:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764121.660607:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1787764121.660608:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1787764121.660612:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1787764121.660615:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1787764121.660617:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:0.0:1787764121.660622:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764121.660625:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764121.660626:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb400.
00000100:00000010:0.0:1787764121.660628:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
02000000:00000001:1.0:1787764121.660631:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787764121.660632:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787764121.660637:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764121.660638:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764121.660640:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764121.660641:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764121.660644:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764121.660646:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787764121.660647:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
02000000:00000010:0.0:1787764121.660651:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787764121.660656:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787764121.660658:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00100000:0.0:1787764121.660661:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832693248/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764121.660669:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:1.0:1787764121.660669:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832693120, portal 25
00000100:00000001:0.0:1787764121.660673:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764121.660676:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000040:0.0:1787764121.660677:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832693248/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.660679:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000100:00000001:0.0:1787764121.660690:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764121.660693:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832693120/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787764121.660694:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764121.660701:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787764121.660704:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764121.660707:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764121.660712:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1787764121.660714:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:0.0:1787764121.660716:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21836
00000100:00000040:1.0:1787764121.660721:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787764121.660722:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764121.660727:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000010:1.0:1787764121.660728:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00080000:0.0:1787764121.660729:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764121.660734:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000200:1.0:1787764121.660735:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832693120, offset 0
00000100:00000001:0.0:1787764121.660736:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764121.660741:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764121.660744:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764121.660745:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764121.660747:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000400:00000200:1.0:1787764121.660749:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000020:00000001:0.0:1787764121.660750:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764121.660753:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787764121.660756:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787764121.660757:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787764121.660760:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:1.0:1787764121.660764:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764121.660765:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764121.660769:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764121.660772:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764121.660773:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787764121.660778:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:1.0:1787764121.660778:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:0.0:1787764121.660781:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764121.660785:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764121.660787:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1787764121.660788:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:0.0:1787764121.660789:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764121.660793:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764121.660794:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787764121.660797:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764121.660797:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:0.0:1787764121.660802:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764121.660805:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000010:0.0:1787764121.660806:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787764121.660811:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000400:00000200:1.0:1787764121.660814:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
02000000:00000001:0.0:1787764121.660815:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764121.660817:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764121.660820:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764121.660823:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1787764121.660824:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58650:58650:256:4294967295] 192.168.203.110@tcp LPNI seq info [58650:58650:8:4294967295]
02000000:00000001:0.0:1787764121.660825:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764121.660831:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000400:00000200:1.0:1787764121.660835:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764121.660836:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764121.660843:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832693376/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787764121.660856:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1787764121.660868:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:1.0:1787764121.660870:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787764121.660871:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764121.660877:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832693376/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:1.0:1787764121.660880:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000100:00000001:0.0:1787764121.660887:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764121.660889:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:1.0:1787764121.660894:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787764121.660895:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787764121.660898:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764121.660902:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000800:00000200:1.0:1787764121.660902:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:0.0:1787764121.660904:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764121.660906:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764121.660910:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764121.660914:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21836
00000800:00000200:1.1:1787764121.660917:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000400:00000200:0.0:1787764121.660918:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764121.660924:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000800:00000200:1.1:1787764121.660925:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000100:00080000:0.0:1787764121.660926:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764121.660932:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764121.660935:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000100:1.0:1787764121.660938:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787764121.660939:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000010:1.0:1787764121.660942:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000100:00000001:0.0:1787764121.660943:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764121.660947:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764121.660949:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787764121.660950:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764121.660952:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000400:00000200:1.0:1787764121.660953:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:0.0:1787764121.660954:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787764121.660958:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787764121.660963:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:1.0:1787764121.660965:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1787764121.660966:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764121.660972:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764121.660974:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:1.0:1787764121.660974:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000020:00000040:0.0:1787764121.660980:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787764121.660981:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764121.660983:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764121.660984:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764121.660987:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:1.0:1787764121.660988:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
02000000:00000001:0.0:1787764121.660990:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764121.660995:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764121.660996:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000008:00100000:0.0:1787764121.660997:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787764121.661002:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787764121.661004:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e1c0 x1874599832693120/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764121.661005:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764121.661012:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787764121.661018:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:1.0:1787764121.661022:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:0.0:1787764121.661023:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764121.661027:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764121.661030:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832693120/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764146 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764121.661031:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764121.661034:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764121.661037:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764121.661044:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:1.0:1787764121.661046:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764121.661050:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764121.661051:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00100000:0.0:1787764121.661059:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832693504/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764121.661059:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764121.661070:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764121.661072:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764121.661075:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316509939
00000100:00000001:0.0:1787764121.661076:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764121.661082:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832693504/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:1.0:1787764121.661085:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c44e0.
00000100:00000001:0.0:1787764121.661088:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764121.661093:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21846)
00000100:00000001:1.0:1787764121.661094:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787764121.661103:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316509939
00000100:00000010:1.0:1787764121.661107:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4060.
00000100:00000001:1.0:1787764121.661111:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787764121.661116:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764121.661120:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764121.661123:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764121.661130:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764121/real 1787764121]  req@ffff9751f876e1c0 x1874599832693120/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764121.661167:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764121.661169:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764121.661172:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764121.661176:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764121.661178:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764121.661182:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764121.661187:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832693120/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764121.661197:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832693120/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.661204:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764121.661206:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764121.661208:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764121.661210:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764121.661215:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832693120/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.661223:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764121.661228:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832693120/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.661236:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764121.661239:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764121.661241:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764121.661242:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764121.661244:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764121.661246:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764121.661249:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764121.661253:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764121.661257:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764121.661261:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832693120/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764121.661271:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832693120:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787764121.661276:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764121.661278:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764121.661281:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832693120/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.661289:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764121.661291:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764121.661294:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764121.661297:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764121.661298:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764121.661300:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787764121.661303:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764121.661306:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
02000000:00000001:1.0:1787764121.661310:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764121.661312:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764121.661314:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:1.0:1787764121.661318:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764121.661319:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764121.661322:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764121.661328:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764121.661329:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764121.661331:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764121.661333:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764121.661335:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764121.661339:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832693248/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.661346:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764121.661347:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764121.661350:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764121.661352:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764121.661354:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764121.661358:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832693248:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764121.661361:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764121.661363:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764121.661365:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764121.661367:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764121.661370:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764121.661372:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764121.661376:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764121.661379:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764121.661382:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764121.661404:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832693248, portal 10
00000100:00000001:1.0:1787764121.661407:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764121.661408:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00000040:1.0:1787764121.661415:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832693248/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.661422:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764121.661425:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764121.661428:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764121.661431:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832693248, offset 0
00000400:00000200:1.0:1787764121.661435:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764121.661439:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764121.661443:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764121.661447:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764121.661450:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764121.661452:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764121.661455:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764121.661458:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58651:58651:256:4294967295] 192.168.203.110@tcp LPNI seq info [58651:58651:8:4294967295]
00000400:00000200:1.0:1787764121.661463:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764121.661470:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764121.661475:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764121.661480:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764121.661484:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764121.661487:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787764121.661492:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764121.661495:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1787764121.661499:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764121.661501:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764121.661504:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764121.661508:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764121.661511:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764121.661513:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764121.661515:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764121.661519:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e580 x1874599832693248/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.661527:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764121.661531:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832693248/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764146 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.661538:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764121.661540:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764121.661543:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764121.661546:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787764121.661550:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764121.661552:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316508704
00000100:00000001:1.0:1787764121.661555:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316509939
00000100:00000001:1.0:1787764121.661557:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316508704
00000100:00000001:1.0:1787764121.661560:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316509939
00000100:00000001:1.0:1787764121.661562:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764121.661564:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764121.661566:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764121.661570:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764121/real 1787764121]  req@ffff9751f876e580 x1874599832693248/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764121.661585:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764121.661587:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764121.661589:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764121.661591:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764121.661593:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764121.661595:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764121.661598:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832693248/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764121.661608:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832693248/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.661615:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764121.661616:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764121.661618:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764121.661620:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764121.661625:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832693248/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.661633:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764121.661637:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832693248/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.661644:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764121.661646:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764121.661647:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764121.661649:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764121.661651:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764121.661664:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764121.661667:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764121.661671:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764121.661673:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764121.661678:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832693248/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764121.661686:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832693248:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764121.661690:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764121.661694:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f0c0 x1874599832693376/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.661701:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764121.661703:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764121.661705:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764121.661707:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764121.661709:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764121.661712:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832693376:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764121.661716:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764121.661717:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764121.661719:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764121.661721:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764121.661725:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9000.
02000000:00000001:1.0:1787764121.661727:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764121.661730:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764121.661732:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764121.661735:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764121.661738:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832693376, portal 10
00000100:00000001:1.0:1787764121.661740:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764121.661742:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049530560 : -115096660021056 : ffff9751f876f0c0)
00000100:00000040:1.0:1787764121.661748:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f0c0 x1874599832693376/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.661755:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764121.661758:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764121.661760:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764121.661763:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832693376, offset 0
00000400:00000200:1.0:1787764121.661767:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764121.661771:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764121.661774:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764121.661777:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764121.661781:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764121.661783:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764121.661786:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764121.661789:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58652:58652:256:4294967295] 192.168.203.110@tcp LPNI seq info [58652:58652:8:4294967295]
00000400:00000200:1.0:1787764121.661793:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764121.661800:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764121.661805:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764121.661808:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764121.661812:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764121.661815:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787764121.661819:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764121.661822:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1787764121.661826:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764121.661829:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764121.661831:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764121.661835:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764121.661838:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764121.661840:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764121.661843:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764121.661846:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f0c0 x1874599832693376/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.661855:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764121.661858:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f0c0 x1874599832693376/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764146 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.661866:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764121.661867:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764121.661870:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764121.661873:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787764121.661876:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764121.661879:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316508692
00000100:00000001:1.0:1787764121.661881:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316509939
00000100:00000001:1.0:1787764121.661884:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316508692
00000100:00000001:1.0:1787764121.661886:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316509939
00000100:00000001:1.0:1787764121.661889:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764121.661891:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764121.661893:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764121.661897:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764121/real 1787764121]  req@ffff9751f876f0c0 x1874599832693376/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764121.661905:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764121.661906:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764121.661908:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764121.661910:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764121.661912:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764121.661914:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764121.661918:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f0c0 x1874599832693376/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764121.661926:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f0c0 x1874599832693376/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.661934:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764121.661936:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764121.661938:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764121.661939:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764121.661942:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f0c0 x1874599832693376/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.661950:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764121.661954:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f0c0 x1874599832693376/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.661962:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764121.661965:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764121.661967:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764121.661968:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764121.661970:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764121.661972:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764121.661975:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764121.661978:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764121.661981:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764121.661985:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f0c0 x1874599832693376/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764121.661994:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832693376:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764121.661998:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764121.662002:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832693504/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.662009:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764121.662011:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764121.662013:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764121.662015:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764121.662017:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764121.662020:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832693504:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764121.662023:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764121.662025:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764121.662027:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764121.662029:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764121.662033:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86ca000.
02000000:00000001:1.0:1787764121.662035:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764121.662038:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764121.662040:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764121.662044:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764121.662047:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832693504, portal 4
00000100:00000001:1.0:1787764121.662049:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764121.662051:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
00000100:00000040:1.0:1787764121.662056:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832693504/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.662063:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764121.662065:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764121.662068:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764121.662070:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832693504, offset 0
00000400:00000200:1.0:1787764121.662074:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764121.662077:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764121.662080:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764121.662083:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764121.662086:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764121.662088:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764121.662091:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764121.662095:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58653:58653:256:4294967295] 192.168.203.110@tcp LPNI seq info [58653:58653:8:4294967295]
00000400:00000200:1.0:1787764121.662099:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764121.662105:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764121.662110:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764121.662112:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764121.662116:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764121.662119:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:1.1:1787764121.662124:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764121.662126:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:1.0:1787764121.662131:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764121.662157:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764121.662161:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764121.662165:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764121.662168:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764121.662171:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764121.662174:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764121.662178:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876ed00 x1874599832693504/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764146 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.662193:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764121.662197:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832693504/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764146 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.662205:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764121.662206:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764121.662209:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764121.662212:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787764121.662215:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764121.662217:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316508710
00000100:00000001:1.0:1787764121.662220:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316509939
00000100:00000001:1.0:1787764121.662222:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316508710
00000100:00000001:1.0:1787764121.662225:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316509939
00000100:00000001:1.0:1787764121.662227:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764121.662229:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764121.662231:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764121.662235:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764121/real 1787764121]  req@ffff9751f876ed00 x1874599832693504/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764121.662244:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764121.662246:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764121.662248:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764121.662250:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764121.662252:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764121.662254:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764121.662257:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832693504/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764121.662267:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832693504/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.662274:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764121.662276:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764121.662277:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764121.662279:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764121.662283:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832693504/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.662291:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764121.662295:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832693504/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.662302:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764121.662304:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764121.662306:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764121.662307:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764121.662309:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764121.662311:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764121.662313:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764121.662316:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764121.662318:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764121.662322:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832693504/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764121.662331:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832693504:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764121.662335:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764121.662337:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764121.662340:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832693248/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.662347:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764121.662348:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764121.662351:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb400.
02000000:00000001:1.0:1787764121.662353:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764121.662355:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764121.662357:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764121.662359:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764121.662361:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014400.
02000000:00000001:1.0:1787764121.662363:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764121.662365:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764121.662367:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:1.0:1787764121.662369:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764121.662371:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764121.662373:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764121.662376:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f0c0 x1874599832693376/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.662384:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764121.662401:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764121.662403:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9000.
02000000:00000001:1.0:1787764121.662406:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764121.662407:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764121.662409:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764121.662411:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764121.662412:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017000.
02000000:00000001:1.0:1787764121.662416:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764121.662417:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764121.662418:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f0c0.
00000100:00000001:1.0:1787764121.662421:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764121.662422:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764121.662424:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764121.662428:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832693504/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764146 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764121.662435:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764121.662436:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764121.662438:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86ca000.
02000000:00000001:1.0:1787764121.662440:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764121.662441:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764121.662442:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787764121.662444:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764121.662445:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:1.0:1787764121.662447:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764121.662449:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764121.662451:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:1.0:1787764121.662453:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764121.662455:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764121.662457:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764121.662460:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764121.662462:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764121.662464:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764121.662465:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764122.300265:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764122.300273:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764122.300275:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764122.300279:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764122.300280:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764122.300289:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764122.300289:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764122.300293:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764122.300296:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764122.300300:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764122.300302:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764122.300305:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764122.300307:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764122.300310:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764122.300312:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764122.300315:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764122.300317:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764122.300320:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764122.300324:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764122.300327:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764122.300330:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764122.300399:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764122.300406:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764122.300411:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764122.684248:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764122.684255:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764122.684257:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764122.684262:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764122.684266:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764122.684269:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764122.684271:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764122.684273:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764122.684275:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764122.940239:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316509939, at 4316509952
00000100:00000010:3.0:1787764122.940253:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4060.
00000100:00000040:2.0:1787764122.940278:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316509939, at 4316509952
00000100:00000010:2.0:1787764122.940285:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c44e0.
00000100:00000001:2.0:1787764123.324235:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764123.324246:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764123.324248:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764123.324249:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764123.324256:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764123.324261:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764123.324261:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764123.324265:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764123.324268:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764123.324271:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764123.324272:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764123.324274:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764123.324276:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764123.324283:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764123.324285:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764123.324340:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764123.324346:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764123.324350:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764123.708267:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764123.708276:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764123.708279:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764123.708285:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764123.708288:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764123.708291:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764123.708293:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764123.708296:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764123.708298:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764124.348229:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764124.348239:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764124.348242:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764124.348248:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764124.348253:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764124.348257:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764124.348260:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764124.348263:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764124.348263:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764124.348271:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764124.348271:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764124.348276:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764124.348279:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764124.348283:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764124.348286:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764124.348288:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764124.348291:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764124.348294:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764124.348295:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764124.348298:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764124.348300:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764124.348347:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764124.348351:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764124.348353:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764124.732252:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764124.732263:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764124.732266:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764124.732272:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764124.732276:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764124.732281:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764124.732283:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764124.732286:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764124.732290:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764125.372238:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764125.372240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764125.372244:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764125.372246:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764125.372251:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764125.372251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764125.372254:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764125.372257:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764125.372258:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764125.372261:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764125.372263:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764125.372268:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764125.372271:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764125.372274:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764125.372277:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764125.372293:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764125.372298:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764125.372302:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764125.756249:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764125.756256:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764125.756259:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764125.756265:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764125.756270:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764125.756273:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764125.756276:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764125.756279:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764125.756283:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764126.396195:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764126.396201:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764126.396206:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764126.396208:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764126.396212:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764126.396217:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764126.396220:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764126.396220:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764126.396224:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764126.396230:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764126.396235:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764126.396238:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.396242:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764126.396245:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.396249:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764126.396310:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764126.396323:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764126.396329:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764126.780257:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:1.0:1787764126.780271:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764126.780276:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:1.0:1787764126.780279:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787764126.780280:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764126.780284:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764126.780290:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:2.0:1787764126.780291:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764126.780297:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787764126.780298:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764126.780304:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:1.0:1787764126.780307:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21841
00000100:00000001:2.0:1787764126.780308:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764126.780312:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.780316:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764126.780320:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764126.780330:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764126.780335:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764126.780339:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764126.780343:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764126.780346:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764126.780349:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764126.780365:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764126.780368:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787764126.780370:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764126.780375:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787764126.780378:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787764126.780381:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787764126.780386:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764126.780389:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764126.780391:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764126.780394:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764126.780397:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787764126.780400:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764126.780403:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764126.780407:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764126.780410:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764126.780413:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764126.780415:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764126.780418:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764126.780425:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932940.
00000020:00000040:1.0:1787764126.780430:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787764126.780434:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764126.780437:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764126.780440:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764126.780444:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764126.780447:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764126.780456:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca000.
00000100:00000001:1.0:1787764126.780463:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764126.780474:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932940 x1874599832693632/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764126.780485:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764126.780490:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764126.780497:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000af90eaa] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932940 x1874599832693632/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764126.780515:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764126.780518:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764126.780523:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787764126.780525:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787764126.780525:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:1.0:1787764126.780529:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764126.780531:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764126.780534:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1787764126.780537:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:2.0:1787764126.780543:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932940 x1874599832693632/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787764126.780544:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21841
00000400:00000200:1.0:1787764126.780555:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764126.780557:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764126.780560:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764126.780562:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
02000000:00000001:2.0:1787764126.780565:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:1.0:1787764126.780567:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
02000000:00000001:2.0:1787764126.780568:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764126.780573:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764126.780575:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00100000:2.0:1787764126.780578:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832693632:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787764126.780580:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764126.780585:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:1.0:1787764126.780586:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764126.780589:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1787764126.780591:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:2.0:1787764126.780592:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764126.780597:0:304484:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:2.0:1787764126.780597:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:1.0:1787764126.780601:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787764126.780604:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000010:2.0:1787764126.780604:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764126.780608:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1787764126.780609:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787764126.780613:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000400:00000200:2.0:1787764126.780618:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:1.0:1787764126.780619:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000400:00000010:2.0:1787764126.780625:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000100:00000040:1.0:1787764126.780626:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764126.780632:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764126.780634:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000010:2.0:1787764126.780637:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000001:1.0:1787764126.780638:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764126.780643:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787764126.780648:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000200:2.0:1787764126.780649:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832693632, portal 25
00000100:00000001:1.0:1787764126.780651:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764126.780656:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787764126.780656:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764126.780661:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497731904 : -115097211819712 : ffff9751d7932940)
02000000:00000001:1.0:1787764126.780664:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764126.780669:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764126.780673:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:2.0:1787764126.780674:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932940 x1874599832693632/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:1.0:1787764126.780677:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787764126.780685:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764126.780692:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764126.780696:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787764126.780699:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764126.780702:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000010:1.0:1787764126.780706:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930780.
00000100:00000200:2.0:1787764126.780708:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832693632, offset 0
00000020:00000040:1.0:1787764126.780712:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764126.780717:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:2.0:1787764126.780718:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
02000000:00000001:1.0:1787764126.780719:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764126.780723:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764126.780724:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764126.780727:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764126.780728:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000010:1.0:1787764126.780735:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9000.
00000400:00000200:2.0:1787764126.780736:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:1.0:1787764126.780743:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764126.780744:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764126.780751:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00100000:1.0:1787764126.780755:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930780 x1874599832693760/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787764126.780757:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764126.780762:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764126.780770:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58654:58654:256:4294967295] 192.168.203.110@tcp LPNI seq info [58654:58654:8:4294967295]
00000100:00000001:1.0:1787764126.780775:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1787764126.780778:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764126.780781:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764126.780791:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3977950] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930780 x1874599832693760/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764126.780791:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764126.780802:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764126.780810:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000100:00000001:1.0:1787764126.780812:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764126.780817:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:2.0:1787764126.780819:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00080000:1.0:1787764126.780825:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000800:00000200:2.0:1787764126.780826:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:1.0:1787764126.780827:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764126.780832:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:2.1:1787764126.780837:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787764126.780838:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:2.1:1787764126.780841:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000100:00080000:1.0:1787764126.780846:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21841
00000800:00000100:2.0:1787764126.780848:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764126.780852:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:1.0:1787764126.780857:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764126.780859:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764126.780865:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:2.0:1787764126.780866:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:1.0:1787764126.780871:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1787764126.780873:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764126.780877:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000040:1.0:1787764126.780878:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764126.780882:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:1.0:1787764126.780883:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764126.780889:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000200:2.0:1787764126.780890:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932940 x1874599832693632/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764126.780892:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764126.780898:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764126.780901:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787764126.780905:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1787764126.780906:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000040:1.0:1787764126.780907:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000100:00000040:2.0:1787764126.780911:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932940 x1874599832693632/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764151 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1787764126.780913:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787764126.780918:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000001:2.0:1787764126.780926:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764126.780927:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764126.780928:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764126.780933:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764126.780935:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764126.780936:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764126.780940:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787764126.780941:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:1.0:1787764126.780947:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00000001:2.0:1787764126.780947:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316515059
00000100:00000010:2.0:1787764126.780953:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7720.
00000100:00080000:1.0:1787764126.780954:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764126.780960:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764126.780962:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787764126.780962:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
02000000:00000001:1.0:1787764126.780965:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764126.780969:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316515059
02000000:00000001:1.0:1787764126.780970:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764126.780976:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000010:2.0:1787764126.780976:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb78a0.
00000008:00100000:1.0:1787764126.780980:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1787764126.780981:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787764126.780985:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000008:00000001:1.0:1787764126.780987:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764126.780989:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764126.780991:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764126.780992:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000010:1.0:1787764126.780997:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932580.
00000100:00100000:2.0:1787764126.780998:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764126/real 1787764126]  req@ffff9751d7932940 x1874599832693632/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787764126.781002:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764126.781009:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764126.781012:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:2.0:1787764126.781014:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:1.0:1787764126.781016:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000001:2.0:1787764126.781017:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:1.0:1787764126.781020:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:2.0:1787764126.781021:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
02000000:00000001:1.0:1787764126.781024:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764126.781025:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764126.781028:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764126.781031:0:257575:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000010:1.0:1787764126.781032:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb400.
00000100:00000200:2.0:1787764126.781035:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932940 x1874599832693632/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764126.781037:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764126.781049:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932580 x1874599832693888/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:2.0:1787764126.781049:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932940 x1874599832693632/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764126.781080:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764126.781081:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764126.781084:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764126.781085:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.781090:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:1.0:1787764126.781092:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d026924] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932580 x1874599832693888/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.781094:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764126.781105:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932940 x1874599832693632/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764126.781107:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764126.781113:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764126.781120:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764126.781122:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000040:2.0:1787764126.781126:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932940 x1874599832693632/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.781131:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764126.781152:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.781155:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764126.781156:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.781158:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764126.781160:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764126.781163:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764126.781168:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00080000:1.0:1787764126.781170:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764126.781171:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764126.781178:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932940 x1874599832693632/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787764126.781179:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787764126.781183:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764126.781187:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764126.781193:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00100000:2.0:1787764126.781198:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832693632:192.168.203.110@tcp:250:kworker.0
00000100:00080000:1.0:1787764126.781200:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21841
00000400:00000200:1.0:1787764126.781210:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764126.781210:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.781216:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000400:00000200:1.0:1787764126.781218:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764126.781222:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764126.781225:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932940 x1874599832693632/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787764126.781228:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764126.781234:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764126.781240:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764126.781240:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764126.781243:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000001:1.0:1787764126.781245:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000010:2.0:1787764126.781246:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
00000020:00000001:1.0:1787764126.781250:0:304484:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:2.0:1787764126.781252:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000040:1.0:1787764126.781253:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787764126.781255:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000001:1.0:1787764126.781258:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764126.781258:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000040:1.0:1787764126.781262:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787764126.781263:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764126.781267:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca000.
00000020:00000001:1.0:1787764126.781268:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
02000000:00000001:2.0:1787764126.781271:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000020:00000001:1.0:1787764126.781272:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
02000000:00000001:2.0:1787764126.781274:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764126.781278:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932940.
00000100:00000040:1.0:1787764126.781280:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764126.781282:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764126.781284:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764126.781285:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.781288:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764126.781289:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764126.781293:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764126.781295:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764126.781298:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1787764126.781299:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:2.0:1787764126.781301:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764126.781305:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:1.0:1787764126.781306:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764126.781307:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1787764126.781312:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764126.781313:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930780 x1874599832693760/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787764126.781314:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764126.781320:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764126.781323:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:1.0:1787764126.781325:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764126.781325:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764126.781331:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:1.0:1787764126.781332:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1787764126.781333:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00100000:1.0:1787764126.781336:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
02000000:00000001:2.0:1787764126.781337:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764126.781342:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832693760:192.168.203.110@tcp:38:kworker.0
00000008:00000001:1.0:1787764126.781343:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764126.781348:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1787764126.781349:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764126.781366:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764126.781368:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764126.781370:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933840.
02000000:00000001:2.0:1787764126.781370:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:1.0:1787764126.781376:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000010:2.0:1787764126.781376:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764126.781381:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764126.781382:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764126.781385:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764126.781386:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764126.781389:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:2.0:1787764126.781390:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
02000000:00000001:1.0:1787764126.781393:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:2.0:1787764126.781395:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
02000000:00000001:1.0:1787764126.781397:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764126.781399:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832693760, portal 10
00000100:00000001:2.0:1787764126.781403:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000010:1.0:1787764126.781404:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8000.
00000100:00000001:2.0:1787764126.781404:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497723264 : -115097211828352 : ffff9751d7930780)
00000100:00000001:1.0:1787764126.781410:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764126.781414:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930780 x1874599832693760/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764126.781421:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933840 x1874599832694016/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764126.781422:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764126.781428:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764126.781432:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764126.781436:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832693760, offset 0
00000100:00000001:1.0:1787764126.781440:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1787764126.781441:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787764126.781446:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764126.781448:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764126.781457:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000040:1.0:1787764126.781458:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000083dd6d32] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933840 x1874599832694016/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764126.781460:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764126.781464:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764126.781469:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764126.781474:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764126.781480:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58655:58655:256:4294967295] 192.168.203.110@tcp LPNI seq info [58655:58655:8:4294967295]
00000100:00000001:1.0:1787764126.781488:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764126.781488:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764126.781495:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:1.0:1787764126.781496:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21851)
00000800:00000200:2.0:1787764126.781501:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764126.781504:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764126.781507:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764126.781509:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787764126.781513:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764126.781515:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1787764126.781518:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764126.781519:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764126.781522:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764126.781524:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764126.781526:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764126.781527:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764126.781529:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764126.781531:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930780 x1874599832693760/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.781537:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764126.781539:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930780 x1874599832693760/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764151 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.781548:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764126.781549:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764126.781551:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764126.781554:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764126.781556:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764126.781558:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316513824
00000100:00000001:2.0:1787764126.781560:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316515059
00000100:00000001:2.0:1787764126.781561:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316513824
00000100:00000001:2.0:1787764126.781563:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316515059
00000100:00000001:2.0:1787764126.781564:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764126.781565:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764126.781567:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764126.781569:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764126/real 1787764126]  req@ffff9751d7930780 x1874599832693760/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764126.781574:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764126.781575:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764126.781576:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764126.781577:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764126.781578:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764126.781579:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764126.781581:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930780 x1874599832693760/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764126.781588:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930780 x1874599832693760/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.781594:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764126.781596:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.781598:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764126.781599:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764126.781603:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930780 x1874599832693760/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.781610:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764126.781615:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930780 x1874599832693760/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.781622:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764126.781624:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.781626:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764126.781627:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.781629:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764126.781641:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764126.781644:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764126.781648:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764126.781650:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764126.781654:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930780 x1874599832693760/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764126.781663:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832693760:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764126.781668:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764126.781671:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932580 x1874599832693888/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.781679:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764126.781680:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764126.781682:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764126.781683:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764126.781685:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764126.781687:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832693888:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764126.781689:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764126.781691:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764126.781692:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764126.781693:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764126.781696:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764126.781697:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764126.781699:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764126.781701:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764126.781703:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764126.781706:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832693888, portal 10
00000100:00000001:2.0:1787764126.781707:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764126.781708:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497730944 : -115097211820672 : ffff9751d7932580)
00000100:00000040:2.0:1787764126.781712:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932580 x1874599832693888/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.781716:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764126.781718:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764126.781719:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764126.781721:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832693888, offset 0
00000400:00000200:2.0:1787764126.781724:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764126.781726:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764126.781729:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764126.781731:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764126.781732:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764126.781734:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764126.781736:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764126.781738:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58656:58656:256:4294967295] 192.168.203.110@tcp LPNI seq info [58656:58656:8:4294967295]
00000400:00000200:2.0:1787764126.781740:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764126.781745:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764126.781748:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764126.781750:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764126.781752:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764126.781754:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787764126.781757:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764126.781759:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1787764126.781762:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764126.781764:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764126.781766:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764126.781768:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764126.781770:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764126.781771:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764126.781773:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764126.781775:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932580 x1874599832693888/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.781780:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764126.781782:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932580 x1874599832693888/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764151 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.781786:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764126.781787:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764126.781789:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764126.781791:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764126.781793:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764126.781795:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316513812
00000100:00000001:2.0:1787764126.781796:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316515059
00000100:00000001:2.0:1787764126.781798:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316513812
00000100:00000001:2.0:1787764126.781799:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316515059
00000100:00000001:2.0:1787764126.781801:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764126.781802:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764126.781803:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764126.781805:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764126/real 1787764126]  req@ffff9751d7932580 x1874599832693888/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764126.781810:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764126.781811:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764126.781812:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764126.781813:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764126.781814:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764126.781816:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764126.781818:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932580 x1874599832693888/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764126.781823:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932580 x1874599832693888/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.781827:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764126.781828:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.781829:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764126.781830:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764126.781832:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932580 x1874599832693888/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.781836:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764126.781838:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932580 x1874599832693888/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.781842:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764126.781844:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.781845:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764126.781845:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.781846:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764126.781847:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764126.781849:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764126.781851:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764126.781852:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764126.781855:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932580 x1874599832693888/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764126.781860:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832693888:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764126.781862:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.781864:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764126.781866:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930780 x1874599832693760/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.781870:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764126.781871:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764126.781872:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764126.781874:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764126.781875:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764126.781876:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764126.781877:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764126.781878:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9000.
02000000:00000001:2.0:1787764126.781880:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764126.781881:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764126.781882:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930780.
00000100:00000001:2.0:1787764126.781884:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764126.781884:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.781885:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764126.781887:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932580 x1874599832693888/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.781891:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764126.781892:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764126.781893:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764126.781894:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764126.781895:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764126.781896:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764126.781897:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764126.781898:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb400.
02000000:00000001:2.0:1787764126.781900:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764126.781901:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764126.781903:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932580.
00000100:00000001:2.0:1787764126.781905:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764126.781906:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.781908:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.781910:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764126.781912:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764126.781914:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764126.781915:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764126.781917:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764126.781921:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933840 x1874599832694016/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.781928:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764126.781930:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764126.781933:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764126.781935:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764126.781937:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764126.781940:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832694016:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764126.781943:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764126.781945:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764126.781946:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764126.781948:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764126.781951:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764126.781953:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764126.781958:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764126.781961:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764126.781964:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764126.781967:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832694016, portal 4
00000100:00000001:2.0:1787764126.781969:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764126.781970:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497735744 : -115097211815872 : ffff9751d7933840)
00000100:00000040:2.0:1787764126.781976:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933840 x1874599832694016/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.781983:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764126.781986:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764126.781988:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764126.781990:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832694016, offset 0
00000400:00000200:2.0:1787764126.781994:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764126.781998:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764126.782002:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764126.782005:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764126.782008:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764126.782011:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764126.782013:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764126.782016:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58657:58657:256:4294967295] 192.168.203.110@tcp LPNI seq info [58657:58657:8:4294967295]
00000400:00000200:2.0:1787764126.782021:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764126.782027:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764126.782032:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764126.782035:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764126.782038:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764126.782040:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787764126.782043:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764126.782045:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1787764126.782047:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764126.782049:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764126.782051:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764126.782054:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764126.782057:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764126.782059:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764126.782061:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764126.782065:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933840 x1874599832694016/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764151 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.782073:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764126.782077:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933840 x1874599832694016/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764151 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.782084:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764126.782085:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764126.782089:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764126.782092:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764126.782096:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764126.782098:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316513829
00000100:00000001:2.0:1787764126.782100:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316515059
00000100:00000001:2.0:1787764126.782103:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316513829
00000100:00000001:2.0:1787764126.782107:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316515059
00000100:00000001:2.0:1787764126.782109:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764126.782110:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764126.782111:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764126.782115:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764126/real 1787764126]  req@ffff9751d7933840 x1874599832694016/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764126.782120:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764126.782121:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764126.782122:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764126.782124:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764126.782125:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764126.782126:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764126.782128:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933840 x1874599832694016/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764126.782152:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933840 x1874599832694016/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.782157:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764126.782159:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.782161:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764126.782163:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764126.782168:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933840 x1874599832694016/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.782175:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764126.782180:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933840 x1874599832694016/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.782187:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764126.782189:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.782191:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764126.782193:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.782195:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764126.782197:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764126.782200:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764126.782203:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764126.782206:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764126.782209:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933840 x1874599832694016/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764126.782217:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832694016:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764126.782221:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.782223:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764126.782226:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933840 x1874599832694016/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764151 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764126.782234:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764126.782235:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764126.782237:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764126.782240:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764126.782242:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764126.782243:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764126.782246:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764126.782247:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8000.
02000000:00000001:2.0:1787764126.782249:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764126.782249:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764126.782251:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933840.
00000100:00000001:2.0:1787764126.782253:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764126.782254:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.782255:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764126.782258:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764126.782259:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764126.782260:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764126.782261:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764127.420241:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764127.420242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764127.420247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764127.420250:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764127.420253:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764127.420257:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764127.420258:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764127.420261:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00100000:2.0:1787764127.420265:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764127.420265:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764127.420272:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764127.420272:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764127.420274:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764127.420276:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764127.420277:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764127.420283:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764127.420283:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764127.420286:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764127.420288:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764127.420290:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764127.420291:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764127.420359:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764127.420363:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764127.420367:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764127.804221:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764127.804229:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764127.804231:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764127.804236:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764127.804240:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764127.804244:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764127.804247:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764127.804249:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764127.804252:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764128.060234:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316515059, at 4316515072
00000100:00000010:3.0:1787764128.060250:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb78a0.
00000100:00000040:2.0:1787764128.060269:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316515059, at 4316515072
00000100:00000010:2.0:1787764128.060277:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7720.
00010000:00000001:3.0:1787764128.444225:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764128.444231:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764128.444232:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764128.444239:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764128.444241:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764128.444243:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764128.444245:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764128.444250:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764128.444252:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764128.444256:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764128.444257:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764128.444263:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764128.444266:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764128.444269:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764128.444271:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764128.444303:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764128.444307:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764128.444311:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764128.828275:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764128.828285:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764128.828288:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764128.828294:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764128.828299:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764128.828304:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764128.828307:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764128.828310:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764128.828313:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764129.468252:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764129.468261:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764129.468262:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764129.468266:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764129.468267:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764129.468274:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764129.468275:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764129.468282:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764129.468284:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764129.468291:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764129.468292:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764129.468300:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764129.468300:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764129.468307:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764129.468309:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764129.468374:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764129.468379:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764129.468383:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764129.852231:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764129.852240:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764129.852243:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764129.852250:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764129.852255:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764129.852259:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764129.852262:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764129.852264:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764129.852268:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764130.492211:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764130.492220:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764130.492223:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764130.492229:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764130.492229:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764130.492235:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764130.492236:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764130.492241:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764130.492244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764130.492247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764130.492251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764130.492254:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764130.492255:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764130.492259:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764130.492264:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764130.492310:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764130.492314:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764130.492317:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764130.876342:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764130.876351:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764130.876353:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764130.876364:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764130.876369:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764130.876372:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764130.876375:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764130.876378:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764130.876381:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764131.516238:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764131.516238:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764131.516244:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764131.516247:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764131.516250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764131.516251:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764131.516252:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764131.516259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764131.516260:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764131.516261:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764131.516267:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764131.516270:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764131.516274:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764131.516276:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764131.516280:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764131.516303:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764131.516308:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764131.516312:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787764131.900233:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764131.900246:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787764131.900248:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764131.900252:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764131.900256:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764131.900258:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764131.900264:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:1.0:1787764131.900269:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21846
00000100:00000001:3.0:1787764131.900269:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764131.900277:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000400:00000200:1.0:1787764131.900282:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764131.900282:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764131.900288:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:1.0:1787764131.900292:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787764131.900294:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764131.900298:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:1.0:1787764131.900299:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787764131.900304:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764131.900306:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764131.900311:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764131.900327:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764131.900330:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764131.900334:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764131.900336:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787764131.900339:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764131.900342:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787764131.900345:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787764131.900348:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787764131.900352:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764131.900355:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764131.900358:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764131.900360:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764131.900364:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787764131.900367:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764131.900370:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764131.900373:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764131.900376:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764131.900379:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764131.900382:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764131.900385:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764131.900393:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933840.
00000020:00000040:1.0:1787764131.900397:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787764131.900401:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764131.900403:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764131.900406:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764131.900410:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764131.900413:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764131.900423:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8400.
00000100:00000001:1.0:1787764131.900429:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764131.900439:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933840 x1874599832694144/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764131.900455:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764131.900459:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764131.900465:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000083dd6d32] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933840 x1874599832694144/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764131.900494:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764131.900497:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764131.900501:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787764131.900503:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764131.900504:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764131.900507:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764131.900510:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21846
00000400:00000200:1.0:1787764131.900514:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764131.900519:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764131.900522:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764131.900525:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764131.900528:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764131.900531:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764131.900534:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764131.900537:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764131.900538:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787764131.900540:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764131.900543:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787764131.900545:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787764131.900547:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787764131.900551:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764131.900553:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764131.900555:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764131.900557:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764131.900559:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787764131.900561:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764131.900564:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764131.900566:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764131.900571:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764131.900573:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764131.900575:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764131.900579:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787764131.900582:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764131.900585:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764131.900589:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932580.
00000020:00000040:1.0:1787764131.900593:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764131.900596:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764131.900598:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764131.900600:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764131.900602:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764131.900604:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764131.900606:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000010:1.0:1787764131.900608:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9400.
00000100:00000001:3.0:1787764131.900612:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764131.900614:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764131.900617:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00100000:1.0:1787764131.900622:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932580 x1874599832694272/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:3.0:1787764131.900628:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933840 x1874599832694144/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764131.900640:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764131.900645:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764131.900649:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:1.0:1787764131.900654:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d026924] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932580 x1874599832694272/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.900654:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764131.900660:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764131.900666:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764131.900668:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764131.900672:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764131.900673:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764131.900680:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00100000:3.0:1787764131.900682:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832694144:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787764131.900683:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764131.900687:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764131.900691:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1787764131.900692:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764131.900698:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21846
02000000:00000001:3.0:1787764131.900699:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000400:00000200:1.0:1787764131.900703:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764131.900706:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764131.900708:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764131.900711:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764131.900713:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764131.900715:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764131.900717:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764131.900720:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764131.900721:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787764131.900723:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764131.900725:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787764131.900727:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787764131.900729:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787764131.900732:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764131.900734:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764131.900736:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764131.900737:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764131.900740:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787764131.900742:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764131.900746:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764131.900747:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764131.900750:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764131.900753:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764131.900755:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764131.900757:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787764131.900759:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764131.900761:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764131.900765:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930780.
00000020:00000040:1.0:1787764131.900767:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764131.900770:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764131.900771:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764131.900773:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764131.900775:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764131.900776:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764131.900780:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cac00.
00000100:00000001:1.0:1787764131.900783:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764131.900788:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930780 x1874599832694400/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764131.900796:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764131.900798:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764131.900804:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3977950] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930780 x1874599832694400/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764131.900813:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764131.900816:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787764131.900821:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787764131.900824:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764131.900827:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787764131.900828:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764131.900830:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764131.900833:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764131.900836:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21846
02000000:00000001:3.0:1787764131.900842:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764131.900843:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:3.0:1787764131.900849:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000400:00000200:1.0:1787764131.900850:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764131.900854:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764131.900858:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000010:3.0:1787764131.900858:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613000.
00000100:00000001:1.0:1787764131.900865:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1787764131.900865:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764131.900869:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764131.900873:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764131.900877:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:3.0:1787764131.900878:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:1.0:1787764131.900879:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787764131.900882:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:3.0:1787764131.900884:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000020:00000040:1.0:1787764131.900885:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787764131.900891:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787764131.900893:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000010:3.0:1787764131.900896:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000040:1.0:1787764131.900897:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764131.900902:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764131.900905:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000200:3.0:1787764131.900905:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832694144, portal 25
00000100:00000001:1.0:1787764131.900911:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764131.900911:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764131.900916:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497735744 : -115097211815872 : ffff9751d7933840)
00000020:00000040:1.0:1787764131.900917:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:1.0:1787764131.900923:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764131.900926:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764131.900929:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:3.0:1787764131.900929:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933840 x1874599832694144/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787764131.900934:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764131.900939:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764131.900943:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764131.900946:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1787764131.900948:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000008:00000001:1.0:1787764131.900951:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764131.900954:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764131.900958:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932940.
00000020:00000040:1.0:1787764131.900961:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787764131.900964:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764131.900965:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764131.900967:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764131.900969:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764131.900970:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764131.900974:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9c00.
00000100:00000040:3.0:1787764131.900975:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787764131.900980:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764131.900982:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787764131.900988:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832694144, offset 0
00000100:00100000:1.0:1787764131.900990:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932940 x1874599832694528/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787764131.901002:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787764131.901003:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764131.901007:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764131.901013:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000af90eaa] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932940 x1874599832694528/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764131.901015:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764131.901026:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764131.901028:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000040:1.0:1787764131.901033:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21856)
00000400:00000200:3.0:1787764131.901036:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764131.901043:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764131.901047:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764131.901062:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764131.901068:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58658:58658:256:4294967295] 192.168.203.110@tcp LPNI seq info [58658:58658:8:4294967295]
00000400:00000200:3.0:1787764131.901072:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764131.901083:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764131.901090:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764131.901096:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787764131.901103:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764131.901107:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787764131.901113:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764131.901117:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787764131.901121:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764131.901125:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787764131.901130:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764131.901169:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764131.901174:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787764131.901177:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787764131.901181:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764131.901186:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933840 x1874599832694144/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.901199:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764131.901203:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933840 x1874599832694144/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764156 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.901212:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764131.901215:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764131.901222:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764131.901227:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764131.901230:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316520180
00000100:00000010:3.0:1787764131.901236:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40360.
00000100:00000001:3.0:1787764131.901243:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787764131.901249:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316520180
00000100:00000010:3.0:1787764131.901253:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40180.
00000100:00000001:3.0:1787764131.901256:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787764131.901260:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764131.901263:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764131.901265:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764131.901270:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764131/real 1787764131]  req@ffff9751d7933840 x1874599832694144/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764131.901280:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764131.901282:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764131.901284:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787764131.901945:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787764131.901947:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787764131.901951:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764131.901956:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933840 x1874599832694144/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764131.901978:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933840 x1874599832694144/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.901985:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764131.901987:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764131.901990:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764131.901992:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764131.901997:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933840 x1874599832694144/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902005:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764131.902010:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933840 x1874599832694144/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902018:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764131.902021:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764131.902022:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764131.902024:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764131.902027:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764131.902029:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764131.902032:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764131.902037:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764131.902040:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764131.902053:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933840 x1874599832694144/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764131.902063:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832694144:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764131.902067:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764131.902070:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764131.902074:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933840 x1874599832694144/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902088:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764131.902091:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764131.902094:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613000.
02000000:00000001:3.0:1787764131.902098:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764131.902100:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764131.902102:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764131.902104:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764131.902106:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8400.
02000000:00000001:3.0:1787764131.902108:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764131.902110:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764131.902111:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933840.
00000100:00000001:3.0:1787764131.902115:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764131.902116:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764131.902118:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764131.902123:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764131.902125:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764131.902126:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764131.902128:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764131.902129:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764131.902154:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932580 x1874599832694272/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902160:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764131.902161:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764131.902164:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764131.902166:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764131.902169:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764131.902172:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832694272:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764131.902176:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764131.902178:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764131.902180:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764131.902182:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764131.902186:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613000.
02000000:00000001:3.0:1787764131.902188:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764131.902194:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764131.902198:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787764131.902203:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787764131.902207:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832694272, portal 10
00000100:00000001:3.0:1787764131.902210:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764131.902211:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497730944 : -115097211820672 : ffff9751d7932580)
00000100:00000040:3.0:1787764131.902218:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932580 x1874599832694272/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902223:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764131.902225:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764131.902227:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787764131.902231:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832694272, offset 0
00000400:00000200:3.0:1787764131.902235:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764131.902240:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764131.902246:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764131.902249:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764131.902253:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764131.902256:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764131.902258:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764131.902261:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58659:58659:256:4294967295] 192.168.203.110@tcp LPNI seq info [58659:58659:8:4294967295]
00000400:00000200:3.0:1787764131.902265:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764131.902272:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764131.902277:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764131.902281:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787764131.902287:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764131.902289:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787764131.902293:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764131.902296:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787764131.902299:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764131.902300:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787764131.902303:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764131.902305:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764131.902308:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787764131.902310:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787764131.902312:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764131.902332:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932580 x1874599832694272/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902339:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764131.902341:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932580 x1874599832694272/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764156 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902346:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764131.902347:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764131.902351:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764131.902353:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764131.902356:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764131.902357:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316518945
00000100:00000001:3.0:1787764131.902360:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316520180
00000100:00000001:3.0:1787764131.902361:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316518945
00000100:00000001:3.0:1787764131.902363:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316520180
00000100:00000001:3.0:1787764131.902364:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764131.902366:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764131.902367:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764131.902370:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764131/real 1787764131]  req@ffff9751d7932580 x1874599832694272/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764131.902375:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764131.902376:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764131.902377:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787764131.902379:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787764131.902380:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787764131.902382:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764131.902384:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932580 x1874599832694272/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764131.902389:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932580 x1874599832694272/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902397:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764131.902398:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764131.902400:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764131.902401:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764131.902403:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932580 x1874599832694272/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902407:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764131.902409:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932580 x1874599832694272/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902413:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764131.902415:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764131.902416:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764131.902417:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764131.902418:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764131.902427:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764131.902429:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764131.902431:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764131.902432:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764131.902435:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932580 x1874599832694272/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764131.902440:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832694272:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764131.902442:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764131.902444:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930780 x1874599832694400/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902449:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764131.902450:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764131.902452:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764131.902453:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764131.902455:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764131.902457:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832694400:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764131.902459:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764131.902460:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764131.902462:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764131.902463:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764131.902465:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612800.
02000000:00000001:3.0:1787764131.902467:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764131.902469:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764131.902470:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787764131.902481:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787764131.902483:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832694400, portal 10
00000100:00000001:3.0:1787764131.902487:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764131.902488:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497723264 : -115097211828352 : ffff9751d7930780)
00000100:00000040:3.0:1787764131.902495:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930780 x1874599832694400/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902501:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764131.902504:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764131.902507:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787764131.902509:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832694400, offset 0
00000400:00000200:3.0:1787764131.902513:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764131.902518:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764131.902522:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764131.902525:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764131.902528:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764131.902531:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764131.902533:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764131.902536:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58660:58660:256:4294967295] 192.168.203.110@tcp LPNI seq info [58660:58660:8:4294967295]
00000400:00000200:3.0:1787764131.902541:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764131.902547:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764131.902553:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764131.902556:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787764131.902559:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764131.902561:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787764131.902564:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764131.902566:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787764131.902568:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764131.902569:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787764131.902572:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764131.902574:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764131.902576:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787764131.902578:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787764131.902580:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764131.902582:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930780 x1874599832694400/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902588:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764131.902590:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930780 x1874599832694400/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764156 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902594:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764131.902595:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764131.902598:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764131.902600:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764131.902602:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764131.902603:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316518933
00000100:00000001:3.0:1787764131.902605:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316520180
00000100:00000001:3.0:1787764131.902606:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316518933
00000100:00000001:3.0:1787764131.902608:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316520180
00000100:00000001:3.0:1787764131.902609:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764131.902611:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764131.902612:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764131.902614:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764131/real 1787764131]  req@ffff9751d7930780 x1874599832694400/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764131.902620:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764131.902620:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764131.902622:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787764131.902623:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787764131.902624:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787764131.902625:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764131.902627:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930780 x1874599832694400/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764131.902633:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930780 x1874599832694400/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902637:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764131.902637:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764131.902639:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764131.902639:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764131.902642:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930780 x1874599832694400/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902646:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764131.902648:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930780 x1874599832694400/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902652:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764131.902654:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764131.902655:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764131.902656:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764131.902657:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764131.902658:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764131.902660:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764131.902662:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764131.902663:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764131.902666:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930780 x1874599832694400/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764131.902671:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832694400:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764131.902674:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764131.902676:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932940 x1874599832694528/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902680:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764131.902680:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764131.902682:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764131.902683:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764131.902684:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764131.902686:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832694528:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764131.902688:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764131.902689:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764131.902690:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764131.902691:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764131.902694:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612c00.
02000000:00000001:3.0:1787764131.902695:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764131.902697:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764131.902699:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787764131.902701:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787764131.902703:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832694528, portal 4
00000100:00000001:3.0:1787764131.902704:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764131.902705:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497731904 : -115097211819712 : ffff9751d7932940)
00000100:00000040:3.0:1787764131.902709:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932940 x1874599832694528/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902714:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764131.902715:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764131.902716:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787764131.902718:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832694528, offset 0
00000400:00000200:3.0:1787764131.902721:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764131.902723:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764131.902726:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764131.902728:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764131.902730:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764131.902731:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764131.902733:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764131.902735:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58661:58661:256:4294967295] 192.168.203.110@tcp LPNI seq info [58661:58661:8:4294967295]
00000400:00000200:3.0:1787764131.902738:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764131.902742:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764131.902745:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764131.902747:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787764131.902749:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764131.902751:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787764131.902753:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764131.902755:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787764131.902758:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764131.902759:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787764131.902761:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764131.902763:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764131.902765:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787764131.902766:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787764131.902768:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764131.902771:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932940 x1874599832694528/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764156 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902779:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764131.902783:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932940 x1874599832694528/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764156 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902790:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764131.902791:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764131.902794:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764131.902797:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764131.902801:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764131.902803:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316518950
00000100:00000001:3.0:1787764131.902807:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316520180
00000100:00000001:3.0:1787764131.902809:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316518950
00000100:00000001:3.0:1787764131.902812:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316520180
00000100:00000001:3.0:1787764131.902814:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764131.902817:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764131.902818:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764131.902823:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764131/real 1787764131]  req@ffff9751d7932940 x1874599832694528/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764131.902831:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764131.902833:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764131.902835:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787764131.902838:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787764131.902840:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787764131.902842:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764131.902845:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932940 x1874599832694528/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764131.902853:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932940 x1874599832694528/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902860:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764131.902870:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764131.902873:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764131.902875:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764131.902878:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932940 x1874599832694528/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902884:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764131.902888:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932940 x1874599832694528/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902896:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764131.902900:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764131.902902:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764131.902903:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764131.902905:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764131.902911:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764131.902915:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764131.902919:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764131.902921:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764131.902926:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932940 x1874599832694528/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764131.902932:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832694528:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764131.902935:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764131.902937:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764131.902939:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932580 x1874599832694272/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902944:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764131.902945:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764131.902946:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613000.
02000000:00000001:3.0:1787764131.902949:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764131.902950:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764131.902950:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764131.902952:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764131.902953:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9400.
02000000:00000001:3.0:1787764131.902955:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764131.902957:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764131.902958:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932580.
00000100:00000001:3.0:1787764131.902961:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764131.902962:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764131.902964:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764131.902967:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930780 x1874599832694400/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.902980:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764131.902981:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764131.902983:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612800.
02000000:00000001:3.0:1787764131.902986:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764131.902987:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764131.902988:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764131.902990:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764131.902992:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cac00.
02000000:00000001:3.0:1787764131.902994:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764131.902995:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764131.902997:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930780.
00000100:00000001:3.0:1787764131.902999:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764131.903001:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764131.903002:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764131.903005:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932940 x1874599832694528/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764156 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764131.903010:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764131.903011:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764131.903012:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612c00.
02000000:00000001:3.0:1787764131.903013:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764131.903014:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764131.903015:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764131.903016:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764131.903017:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:3.0:1787764131.903019:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764131.903019:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764131.903020:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932940.
00000100:00000001:3.0:1787764131.903021:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764131.903022:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764131.903023:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764131.903026:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764131.903027:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764131.903028:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764131.903029:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764132.540213:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764132.540220:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764132.540222:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764132.540223:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764132.540227:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764132.540228:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764132.540234:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764132.540235:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764132.540238:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764132.540238:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764132.540244:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764132.540245:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764132.540247:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764132.540249:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764132.540251:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764132.540252:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764132.540256:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764132.540259:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764132.540260:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764132.540262:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764132.540264:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764132.540324:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764132.540327:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764132.540330:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764132.924340:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764132.924351:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764132.924354:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764132.924362:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764132.924367:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764132.924373:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764132.924376:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764132.924379:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764132.924383:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764133.180184:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316520180, at 4316520192
00000100:00000010:3.0:1787764133.180203:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40180.
00000100:00000040:2.0:1787764133.180251:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316520180, at 4316520192
00000100:00000010:2.0:1787764133.180264:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40360.
00000100:00000001:2.0:1787764133.564219:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764133.564219:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764133.564226:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764133.564227:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764133.564230:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764133.564233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764133.564235:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764133.564236:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764133.564240:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764133.564242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764133.564244:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764133.564245:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764133.564248:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764133.564251:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764133.564254:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764133.564275:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764133.564281:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764133.564284:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764133.948193:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764133.948200:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764133.948202:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764133.948207:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764133.948210:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764133.948212:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764133.948215:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764133.948217:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764133.948219:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764134.588219:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764134.588221:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764134.588227:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764134.588227:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764134.588230:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764134.588231:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764134.588234:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764134.588236:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764134.588239:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764134.588242:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764134.588247:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764134.588249:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764134.588250:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764134.588252:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764134.588253:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764134.588256:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764134.588258:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764134.588261:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764134.588262:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764134.588266:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764134.588267:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764134.588336:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764134.588341:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764134.588344:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764134.972245:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764134.972253:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764134.972256:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764134.972263:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764134.972268:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764134.972272:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764134.972276:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764134.972278:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764134.972281:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764135.612214:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764135.612220:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764135.612221:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764135.612226:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764135.612226:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764135.612233:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764135.612235:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764135.612237:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764135.612238:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764135.612242:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764135.612245:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764135.612246:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764135.612248:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764135.612249:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764135.612254:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764135.612288:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764135.612307:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764135.612310:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764135.996204:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764135.996211:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764135.996213:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764135.996217:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764135.996220:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764135.996223:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764135.996225:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764135.996227:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764135.996228:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764136.636230:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764136.636231:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764136.636236:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764136.636239:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764136.636243:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764136.636243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764136.636246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764136.636250:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764136.636251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764136.636253:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764136.636258:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764136.636263:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764136.636266:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764136.636269:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764136.636273:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764136.636309:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764136.636313:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764136.636317:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764137.020219:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764137.020227:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764137.020230:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764137.020235:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764137.020240:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:2.0:1787764137.020245:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764137.020245:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.020251:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:2.0:1787764137.020252:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787764137.020256:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1787764137.020257:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1787764137.020260:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764137.020263:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764137.020264:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764137.020272:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21851
00000400:00000200:2.0:1787764137.020290:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764137.020298:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764137.020302:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764137.020305:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764137.020308:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764137.020310:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764137.020312:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764137.020316:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764137.020318:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787764137.020320:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764137.020323:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787764137.020325:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787764137.020328:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787764137.020330:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764137.020331:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764137.020333:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764137.020334:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764137.020336:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787764137.020339:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764137.020340:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764137.020342:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764137.020345:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764137.020347:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764137.020348:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764137.020350:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764137.020356:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658780.
00000020:00000040:2.0:1787764137.020359:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787764137.020362:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764137.020363:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764137.020365:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764137.020367:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764137.020370:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764137.020377:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5b9800.
00000100:00000001:2.0:1787764137.020382:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764137.020389:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658780 x1874599832694656/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764137.020397:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764137.020400:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764137.020404:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000744875b5] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658780 x1874599832694656/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764137.020420:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764137.020422:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764137.020424:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787764137.020426:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764137.020427:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764137.020429:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764137.020431:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21851
00000400:00000200:2.0:1787764137.020434:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764137.020436:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764137.020437:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764137.020439:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764137.020441:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764137.020442:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764137.020444:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764137.020445:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764137.020446:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:2.0:1787764137.020448:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764137.020455:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:2.0:1787764137.020457:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:2.0:1787764137.020459:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:3.0:1787764137.020460:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1787764137.020462:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764137.020466:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764137.020467:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764137.020470:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764137.020472:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764137.020476:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764137.020481:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00000040:3.0:1787764137.020482:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832694656/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787764137.020488:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764137.020496:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764137.020501:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764137.020503:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764137.020507:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764137.020508:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764137.020511:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764137.020513:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764137.020516:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764137.020517:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1787764137.020522:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:2.0:1787764137.020523:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787764137.020528:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764137.020531:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832694656:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764137.020532:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764137.020538:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000100:00000001:3.0:1787764137.020540:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:2.0:1787764137.020541:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764137.020545:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764137.020547:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764137.020547:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1787764137.020552:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764137.020553:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764137.020553:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764137.020556:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764137.020558:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764137.020562:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bb800.
00000100:00000001:2.0:1787764137.020566:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764137.020568:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612c00.
00000100:00100000:2.0:1787764137.020573:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832694784/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1787764137.020574:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764137.020583:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1787764137.020586:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764137.020587:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764137.020594:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832694784/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1787764137.020596:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000100:00000001:2.0:1787764137.020605:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764137.020608:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:3.0:1787764137.020608:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00080000:2.0:1787764137.020614:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1787764137.020615:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764137.020616:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764137.020619:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:3.0:1787764137.020619:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832694656, portal 25
00000100:00000001:3.0:1787764137.020625:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00080000:2.0:1787764137.020626:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21851
00000100:00000001:3.0:1787764137.020630:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000400:00000200:2.0:1787764137.020636:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:3.0:1787764137.020642:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832694656/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764162 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764137.020643:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764137.020649:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764137.020657:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764137.020661:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1787764137.020662:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764137.020669:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000040:3.0:1787764137.020671:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787764137.020672:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764137.020675:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764137.020677:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000400:00000010:3.0:1787764137.020677:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000020:00000001:2.0:1787764137.020681:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764137.020684:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000100:00000200:3.0:1787764137.020685:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832694656, offset 0
00000020:00000001:2.0:1787764137.020686:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:2.0:1787764137.020692:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:2.0:1787764137.020694:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764137.020696:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787764137.020700:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000040:2.0:1787764137.020701:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764137.020706:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764137.020708:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:2.0:1787764137.020710:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764137.020713:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764137.020714:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:2.0:1787764137.020715:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764137.020719:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764137.020722:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764137.020723:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764137.020726:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:3.0:1787764137.020726:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000008:00000001:2.0:1787764137.020732:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764137.020735:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764137.020737:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000010:2.0:1787764137.020739:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000020:00000040:2.0:1787764137.020744:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000400:00000200:3.0:1787764137.020744:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
02000000:00000001:2.0:1787764137.020748:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764137.020752:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764137.020753:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1787764137.020754:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764137.020758:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764137.020760:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764137.020762:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
02000000:00000010:2.0:1787764137.020764:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5ba000.
00000400:00000200:3.0:1787764137.020771:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58662:58662:256:4294967295] 192.168.203.110@tcp LPNI seq info [58662:58662:8:4294967295]
00000400:00000200:3.0:1787764137.020777:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764137.020778:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764137.020786:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832694912/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787764137.020796:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1787764137.020800:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764137.020803:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764137.020807:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832694912/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764137.020813:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764137.020815:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000200:3.0:1787764137.020820:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000040:2.0:1787764137.020821:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000010:3.0:1787764137.020830:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000100:00080000:2.0:1787764137.020831:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764137.020837:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787764137.020838:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764137.020839:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764137.020845:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:3.0:1787764137.020845:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00080000:2.0:1787764137.020850:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21851
00000800:00000200:3.0:1787764137.020853:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000400:00000200:2.0:1787764137.020856:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764137.020861:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764137.020864:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000200:3.1:1787764137.020867:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:2.0:1787764137.020869:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764137.020873:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764137.020874:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764137.020875:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764137.020877:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764137.020877:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787764137.020878:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764137.020880:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787764137.020884:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000800:00000200:3.1:1787764137.020884:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000020:00000001:2.0:1787764137.020890:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000800:00000100:3.0:1787764137.020896:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000040:2.0:1787764137.020898:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000010:3.0:1787764137.020902:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000100:00000001:2.0:1787764137.020903:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764137.020909:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:3.0:1787764137.020911:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1787764137.020913:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764137.020920:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000400:00000200:3.0:1787764137.020921:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:2.0:1787764137.020928:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:3.0:1787764137.020930:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000100:00000001:2.0:1787764137.020934:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764137.020935:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
02000000:00000001:2.0:1787764137.020937:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764137.020941:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764137.020943:0:257575:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:2.0:1787764137.020944:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764137.020947:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764137.020949:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000200:3.0:1787764137.020952:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832694656/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764162 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:2.0:1787764137.020953:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764137.020957:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764137.020962:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
00000020:00000040:2.0:1787764137.020966:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:2.0:1787764137.020972:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764137.020974:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764137.020975:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787764137.020976:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764137.020979:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764137.020981:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764137.020984:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832694656/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764162 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1787764137.020985:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bbc00.
00000100:00000001:2.0:1787764137.020990:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764137.020999:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832695040/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764137.021002:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764137.021007:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764137.021012:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764137.021014:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764137.021016:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764137.021018:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832695040/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764137.021024:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764137.021029:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316525299
00000100:00000001:2.0:1787764137.021037:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764137.021040:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40180.
00000100:00000040:2.0:1787764137.021041:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21861)
00000100:00000001:3.0:1787764137.021049:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787764137.021059:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316525299
00000100:00000010:3.0:1787764137.021063:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d403c0.
00000100:00000001:3.0:1787764137.021066:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787764137.021069:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764137.021072:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764137.021075:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764137.021080:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764137/real 1787764137]  req@ffff9751f8658780 x1874599832694656/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764137.021090:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764137.021092:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764137.021094:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787764137.021096:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787764137.021099:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787764137.021101:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764137.021105:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832694656/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764137.021113:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832694656/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.021120:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764137.021122:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.021124:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764137.021127:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764137.021153:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832694656/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.021163:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764137.021168:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832694656/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.021177:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764137.021180:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.021182:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764137.021183:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.021185:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764137.021187:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764137.021190:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764137.021194:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764137.021196:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764137.021201:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832694656/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764137.021209:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832694656:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764137.021213:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.021216:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764137.021219:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832694656/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.021226:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764137.021228:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764137.021231:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612c00.
02000000:00000001:3.0:1787764137.021235:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764137.021237:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764137.021239:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764137.021241:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764137.021244:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:3.0:1787764137.021248:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764137.021250:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764137.021253:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:3.0:1787764137.021256:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764137.021257:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.021259:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.021266:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764137.021268:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764137.021270:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764137.021271:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764137.021273:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764137.021277:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832694784/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.021299:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764137.021301:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764137.021303:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764137.021306:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764137.021308:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764137.021311:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832694784:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764137.021314:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764137.021316:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764137.021318:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764137.021320:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764137.021324:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612c00.
02000000:00000001:3.0:1787764137.021326:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764137.021329:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764137.021332:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787764137.021338:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787764137.021341:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832694784, portal 10
00000100:00000001:3.0:1787764137.021343:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764137.021344:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:3.0:1787764137.021351:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832694784/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764162 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.021357:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764137.021360:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764137.021363:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787764137.021366:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832694784, offset 0
00000400:00000200:3.0:1787764137.021371:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764137.021375:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764137.021379:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764137.021382:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764137.021385:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764137.021388:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764137.021391:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764137.021394:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58663:58663:256:4294967295] 192.168.203.110@tcp LPNI seq info [58663:58663:8:4294967295]
00000400:00000200:3.0:1787764137.021399:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764137.021405:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764137.021411:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764137.021414:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787764137.021418:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764137.021421:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787764137.021427:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764137.021429:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787764137.021433:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764137.021435:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787764137.021438:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764137.021442:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764137.021446:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787764137.021448:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787764137.021450:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764137.021455:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832694784/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764162 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.021463:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764137.021466:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832694784/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764162 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.021473:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764137.021474:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764137.021478:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764137.021483:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764137.021486:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764137.021488:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316524064
00000100:00000001:3.0:1787764137.021491:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316525299
00000100:00000001:3.0:1787764137.021493:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316524064
00000100:00000001:3.0:1787764137.021496:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316525299
00000100:00000001:3.0:1787764137.021497:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764137.021500:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764137.021502:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764137.021506:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764137/real 1787764137]  req@ffff9751f865ad00 x1874599832694784/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764137.021515:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764137.021517:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764137.021519:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787764137.021521:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787764137.021523:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787764137.021525:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764137.021529:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832694784/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764137.021538:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832694784/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.021544:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764137.021546:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.021548:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764137.021549:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764137.021553:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832694784/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.021559:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764137.021564:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832694784/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.021571:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764137.021574:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.021576:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764137.021578:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.021580:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764137.021592:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764137.021594:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764137.021598:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764137.021601:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764137.021605:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832694784/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764137.021613:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832694784:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764137.021617:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764137.021620:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832694912/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.021627:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764137.021628:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764137.021631:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764137.021633:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764137.021635:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764137.021638:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832694912:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764137.021642:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764137.021644:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764137.021646:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764137.021648:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764137.021652:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612800.
02000000:00000001:3.0:1787764137.021654:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764137.021657:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764137.021660:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787764137.021663:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787764137.021666:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832694912, portal 10
00000100:00000001:3.0:1787764137.021668:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764137.021670:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:3.0:1787764137.021676:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832694912/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764162 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.021682:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764137.021685:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764137.021687:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787764137.021689:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832694912, offset 0
00000400:00000200:3.0:1787764137.021694:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764137.021698:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764137.021702:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764137.021705:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764137.021708:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764137.021710:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764137.021712:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764137.021716:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58664:58664:256:4294967295] 192.168.203.110@tcp LPNI seq info [58664:58664:8:4294967295]
00000400:00000200:3.0:1787764137.021720:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764137.021727:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764137.021731:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764137.021735:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787764137.021739:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764137.021741:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787764137.021746:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764137.021748:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787764137.021752:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764137.021755:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787764137.021758:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764137.021761:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764137.021764:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787764137.021767:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787764137.021770:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764137.021775:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832694912/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764162 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.021783:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764137.021786:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832694912/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764162 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.021793:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764137.021795:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764137.021799:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764137.021802:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764137.021805:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764137.021808:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316524052
00000100:00000001:3.0:1787764137.021810:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316525299
00000100:00000001:3.0:1787764137.021813:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316524052
00000100:00000001:3.0:1787764137.021815:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316525299
00000100:00000001:3.0:1787764137.021817:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764137.021820:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764137.021822:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764137.021826:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764137/real 1787764137]  req@ffff9751f8659a40 x1874599832694912/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764137.021834:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764137.021835:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764137.021837:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787764137.021840:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787764137.021842:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787764137.021844:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764137.021847:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832694912/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764137.021856:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832694912/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.021863:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764137.021865:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.021866:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764137.021868:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764137.021871:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832694912/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.021878:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764137.021882:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832694912/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.021889:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764137.021891:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.021893:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764137.021895:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.021897:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764137.021898:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764137.021901:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764137.021904:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764137.021906:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764137.021911:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832694912/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764137.021919:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832694912:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764137.021923:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764137.021926:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832695040/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.021933:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764137.021935:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764137.021938:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764137.021940:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764137.021942:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764137.021945:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832695040:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764137.021949:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764137.021951:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764137.021952:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764137.021954:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764137.021958:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613000.
02000000:00000001:3.0:1787764137.021961:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764137.021964:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764137.021967:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787764137.021970:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787764137.021973:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832695040, portal 4
00000100:00000001:3.0:1787764137.021975:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764137.021976:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:3.0:1787764137.021982:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832695040/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764162 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.021989:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764137.021992:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764137.021995:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787764137.021997:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832695040, offset 0
00000400:00000200:3.0:1787764137.022001:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764137.022004:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764137.022009:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764137.022012:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764137.022014:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764137.022017:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764137.022019:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764137.022022:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58665:58665:256:4294967295] 192.168.203.110@tcp LPNI seq info [58665:58665:8:4294967295]
00000400:00000200:3.0:1787764137.022026:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764137.022033:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764137.022038:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764137.022041:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787764137.022045:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764137.022048:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787764137.022052:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764137.022055:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787764137.022059:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764137.022061:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787764137.022064:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764137.022067:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764137.022071:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787764137.022073:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787764137.022075:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764137.022079:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832695040/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764162 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.022087:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764137.022090:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832695040/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764162 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.022097:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764137.022098:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764137.022102:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764137.022105:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764137.022108:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764137.022111:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316524069
00000100:00000001:3.0:1787764137.022113:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316525299
00000100:00000001:3.0:1787764137.022115:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316524069
00000100:00000001:3.0:1787764137.022118:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316525299
00000100:00000001:3.0:1787764137.022120:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764137.022122:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764137.022124:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764137.022128:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764137/real 1787764137]  req@ffff9751f8658000 x1874599832695040/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764137.022157:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764137.022158:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764137.022160:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787764137.022163:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787764137.022165:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787764137.022167:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764137.022171:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832695040/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764137.022180:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832695040/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.022187:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764137.022188:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.022190:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764137.022192:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764137.022196:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832695040/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.022202:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764137.022206:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832695040/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.022214:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764137.022216:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.022218:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764137.022220:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.022221:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764137.022223:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764137.022226:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764137.022229:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764137.022232:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764137.022236:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832695040/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764137.022245:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832695040:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764137.022249:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.022251:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764137.022255:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832694784/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.022261:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764137.022263:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764137.022265:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612c00.
02000000:00000001:3.0:1787764137.022268:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764137.022270:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764137.022271:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764137.022273:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764137.022275:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:3.0:1787764137.022278:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764137.022297:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764137.022300:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:3.0:1787764137.022303:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764137.022305:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.022307:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764137.022311:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832694912/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.022317:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764137.022319:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764137.022321:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612800.
02000000:00000001:3.0:1787764137.022323:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764137.022325:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764137.022326:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764137.022328:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764137.022330:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:3.0:1787764137.022333:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764137.022335:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764137.022336:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:3.0:1787764137.022338:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764137.022340:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.022341:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764137.022345:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832695040/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764162 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764137.022351:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764137.022353:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764137.022355:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613000.
02000000:00000001:3.0:1787764137.022357:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764137.022359:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764137.022360:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764137.022362:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764137.022364:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:3.0:1787764137.022366:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764137.022368:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764137.022369:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:3.0:1787764137.022371:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764137.022373:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.022375:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764137.022379:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764137.022381:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764137.022383:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764137.022384:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764137.660234:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764137.660234:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764137.660241:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764137.660242:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764137.660244:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764137.660250:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764137.660253:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764137.660254:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764137.660259:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764137.660259:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764137.660264:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764137.660265:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764137.660267:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764137.660272:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764137.660272:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764137.660291:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764137.660294:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764137.660297:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764137.660300:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764137.660306:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764137.660308:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764137.660342:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764137.660346:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764137.660348:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764138.044238:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764138.044247:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764138.044250:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764138.044256:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764138.044260:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764138.044265:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764138.044268:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764138.044271:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764138.044293:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764138.300196:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316525299, at 4316525312
00000100:00000010:3.0:1787764138.300209:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d403c0.
00000100:00000040:2.0:1787764138.300252:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316525299, at 4316525312
00000100:00000010:2.0:1787764138.300262:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40180.
00000100:00000001:2.0:1787764138.684254:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764138.684259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764138.684264:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764138.684264:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764138.684283:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764138.684290:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764138.684291:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764138.684295:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764138.684297:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764138.684302:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764138.684303:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764138.684306:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764138.684309:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764138.684314:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764138.684317:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764138.684354:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764138.684357:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764138.684360:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764139.068264:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764139.068286:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764139.068289:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764139.068294:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764139.068298:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764139.068301:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764139.068303:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764139.068305:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764139.068308:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764139.708221:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764139.708221:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764139.708227:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764139.708229:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764139.708232:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764139.708234:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764139.708235:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764139.708240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764139.708242:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764139.708242:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764139.708249:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764139.708252:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764139.708255:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764139.708258:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764139.708272:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764139.708291:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764139.708297:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764139.708301:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764140.092228:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764140.092235:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764140.092237:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764140.092243:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764140.092248:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764140.092253:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764140.092256:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764140.092270:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764140.092273:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764140.732285:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764140.732292:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764140.732294:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764140.732298:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764140.732299:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764140.732307:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764140.732308:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764140.732315:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764140.732318:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764140.732320:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764140.732321:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764140.732327:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764140.732330:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764140.732331:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764140.732335:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764140.732406:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764140.732411:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764140.732415:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764141.116342:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764141.116350:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764141.116352:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764141.116359:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764141.116364:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764141.116367:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764141.116370:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764141.116373:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764141.116377:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764141.756242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764141.756243:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764141.756262:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764141.756264:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764141.756266:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764141.756274:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764141.756275:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764141.756277:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764141.756282:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764141.756283:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764141.756287:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764141.756289:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764141.756294:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764141.756297:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764141.756300:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764141.756325:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764141.756328:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764141.756331:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764142.140256:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764142.140264:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764142.140266:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764142.140270:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:3.0:1787764142.140270:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764142.140276:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:1.0:1787764142.140277:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764142.140280:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764142.140281:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764142.140285:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764142.140285:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764142.140289:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764142.140291:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764142.140293:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764142.140298:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21856
00000400:00000200:1.0:1787764142.140304:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764142.140311:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764142.140315:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764142.140320:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764142.140323:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764142.140326:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764142.140336:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764142.140340:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764142.140342:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787764142.140345:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764142.140349:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787764142.140352:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787764142.140355:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787764142.140358:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764142.140360:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764142.140362:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764142.140365:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764142.140368:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787764142.140371:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764142.140374:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764142.140377:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764142.140380:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764142.140383:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764142.140386:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764142.140388:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764142.140397:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933480.
00000020:00000040:1.0:1787764142.140401:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787764142.140405:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764142.140407:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764142.140410:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764142.140412:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764142.140416:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764142.140433:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb000.
00000100:00000001:1.0:1787764142.140440:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764142.140449:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933480 x1874599832695168/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764142.140459:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764142.140463:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764142.140468:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3c2c33c] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933480 x1874599832695168/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764142.140489:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764142.140493:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764142.140496:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787764142.140498:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764142.140499:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764142.140502:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764142.140505:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21856
00000400:00000200:1.0:1787764142.140510:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764142.140514:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764142.140517:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787764142.140520:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:1.0:1787764142.140522:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764142.140525:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764142.140526:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764142.140528:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764142.140532:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000040:3.0:1787764142.140537:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933480 x1874599832695168/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764142.140538:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764142.140545:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764142.140548:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787764142.140553:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787764142.140553:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764142.140555:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1787764142.140558:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
02000000:00000001:3.0:1787764142.140559:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764142.140563:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:1.0:1787764142.140564:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:3.0:1787764142.140566:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764142.140572:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00100000:3.0:1787764142.140572:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832695168:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764142.140580:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:1.0:1787764142.140583:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:3.0:1787764142.140584:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764142.140587:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764142.140588:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1787764142.140591:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:1.0:1787764142.140594:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000010:3.0:1787764142.140596:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613000.
00000100:00000001:1.0:1787764142.140600:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:3.0:1787764142.140600:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1787764142.140606:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000400:00000200:3.0:1787764142.140608:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764142.140613:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000100:00080000:1.0:1787764142.140614:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764142.140619:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764142.140620:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
02000000:00000001:1.0:1787764142.140622:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:3.0:1787764142.140628:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832695168, portal 25
02000000:00000001:1.0:1787764142.140630:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764142.140631:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764142.140635:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497734784 : -115097211816832 : ffff9751d7933480)
02000000:00000001:1.0:1787764142.140637:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764142.140642:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764142.140647:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000040:3.0:1787764142.140647:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933480 x1874599832695168/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764167 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:1.0:1787764142.140655:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764142.140660:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764142.140667:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930000.
00000100:00000001:3.0:1787764142.140668:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000040:1.0:1787764142.140671:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000100:00000040:3.0:1787764142.140675:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
02000000:00000001:1.0:1787764142.140677:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764142.140679:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764142.140681:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764142.140682:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:3.0:1787764142.140682:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
02000000:00000001:1.0:1787764142.140685:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764142.140688:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832695168, offset 0
02000000:00000010:1.0:1787764142.140692:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9c00.
00000400:00000200:3.0:1787764142.140697:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787764142.140698:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764142.140704:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930000 x1874599832695296/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764142.140712:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1787764142.140715:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764142.140716:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764142.140722:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6d90274] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930000 x1874599832695296/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764142.140725:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764142.140734:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1787764142.140736:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764142.140739:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00080000:1.0:1787764142.140740:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764142.140744:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:3.0:1787764142.140744:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1787764142.140748:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1787764142.140750:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00080000:1.0:1787764142.140751:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1787764142.140757:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58666:58666:256:4294967295] 192.168.203.110@tcp LPNI seq info [58666:58666:8:4294967295]
00000100:00000001:1.0:1787764142.140758:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764142.140763:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21856
00000400:00000200:3.0:1787764142.140764:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764142.140770:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764142.140778:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1787764142.140781:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00080000:1.0:1787764142.140782:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764142.140789:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764142.140794:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1787764142.140795:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000040:1.0:1787764142.140798:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764142.140801:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764142.140803:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764142.140804:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000800:00000010:3.0:1787764142.140804:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000020:00000001:1.0:1787764142.140806:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764142.140809:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787764142.140812:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787764142.140813:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:3.0:1787764142.140818:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:1.0:1787764142.140819:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764142.140822:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764142.140826:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764142.140828:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:3.0:1787764142.140828:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000020:00000040:1.0:1787764142.140831:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787764142.140836:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764142.140840:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787764142.140841:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:1.0:1787764142.140842:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764142.140845:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000200:3.1:1787764142.140847:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
02000000:00000001:1.0:1787764142.140849:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764142.140854:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000100:3.0:1787764142.140857:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000008:00100000:1.0:1787764142.140858:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000800:00000010:3.0:1787764142.140861:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000008:00000001:1.0:1787764142.140862:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764142.140866:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764142.140869:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930b40.
00000400:00000200:3.0:1787764142.140869:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:1.0:1787764142.140875:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000400:00000200:3.0:1787764142.140878:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:1.0:1787764142.140880:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764142.140883:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764142.140884:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000100:00000001:1.0:1787764142.140885:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764142.140888:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:3.0:1787764142.140889:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
02000000:00000001:1.0:1787764142.140891:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764142.140895:0:257575:0:(events.c:34:request_out_callback()) Process entered
02000000:00000010:1.0:1787764142.140896:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cac00.
00000100:00000001:1.0:1787764142.140900:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764142.140902:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933480 x1874599832695168/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764167 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764142.140907:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930b40 x1874599832695424/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764142.140922:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787764142.140923:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764142.140926:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764142.140929:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933480 x1874599832695168/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764167 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787764142.140933:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022d7f91a] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930b40 x1874599832695424/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.140945:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764142.140949:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764142.140949:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:1.0:1787764142.140953:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787764142.140956:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1787764142.140957:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764142.140958:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764142.140963:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787764142.140965:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1787764142.140965:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:1.0:1787764142.140969:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764142.140970:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316530419
00000100:00000001:1.0:1787764142.140972:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764142.140976:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21856
00000100:00000010:3.0:1787764142.140977:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d403c0.
00000400:00000200:1.0:1787764142.140981:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764142.140986:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787764142.140992:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316530419
00000400:00000200:1.0:1787764142.140994:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000010:3.0:1787764142.140996:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40f00.
00000100:00080000:1.0:1787764142.140998:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787764142.141001:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000040:1.0:1787764142.141002:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764142.141007:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764142.141007:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1787764142.141012:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764142.141012:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764142.141017:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764142.141018:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000020:00000001:1.0:1787764142.141021:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764142.141024:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787764142.141025:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00100000:3.0:1787764142.141027:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764142/real 1787764142]  req@ffff9751d7933480 x1874599832695168/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787764142.141028:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787764142.141033:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787764142.141038:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:1.0:1787764142.141045:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000001:3.0:1787764142.141046:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:1.0:1787764142.141048:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000001:3.0:1787764142.141049:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:1.0:1787764142.141053:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000010:3.0:1787764142.141054:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000100:00000001:1.0:1787764142.141058:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:3.0:1787764142.141059:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000020:00000040:1.0:1787764142.141062:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000400:00000010:3.0:1787764142.141063:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00080000:1.0:1787764142.141068:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764142.141068:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:1.0:1787764142.141072:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764142.141074:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933480 x1874599832695168/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787764142.141075:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764142.141078:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764142.141083:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764142.141087:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764142.141090:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000200:3.0:1787764142.141091:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933480 x1874599832695168/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:1.0:1787764142.141095:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764142.141099:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764142.141106:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7931680.
00000100:00000001:3.0:1787764142.141106:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764142.141109:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:1.0:1787764142.141110:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:3.0:1787764142.141113:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:1.0:1787764142.141114:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1787764142.141116:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:1.0:1787764142.141117:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764142.141120:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764142.141122:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00100000:3.0:1787764142.141123:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933480 x1874599832695168/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787764142.141124:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764142.141129:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9400.
00000100:00000001:1.0:1787764142.141152:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764142.141158:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7931680 x1874599832695552/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764142.141158:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764142.141168:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933480 x1874599832695168/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764142.141191:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764142.141192:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764142.141195:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764142.141197:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764142.141199:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764142.141202:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764142.141203:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d4255d6] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7931680 x1874599832695552/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141205:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764142.141208:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764142.141213:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:1.0:1787764142.141218:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764142.141220:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000040:1.0:1787764142.141223:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21866)
00000100:00000001:3.0:1787764142.141224:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764142.141231:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933480 x1874599832695168/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764142.141237:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832695168:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764142.141241:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764142.141243:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764142.141262:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933480 x1874599832695168/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141267:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764142.141269:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764142.141271:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613000.
02000000:00000001:3.0:1787764142.141274:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764142.141275:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764142.141277:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764142.141278:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764142.141281:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb000.
02000000:00000001:3.0:1787764142.141284:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764142.141285:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764142.141286:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933480.
00000100:00000001:3.0:1787764142.141290:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764142.141291:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764142.141292:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764142.141297:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764142.141298:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764142.141300:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764142.141301:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764142.141302:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764142.141304:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930000 x1874599832695296/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141309:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764142.141311:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764142.141312:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764142.141314:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764142.141315:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764142.141317:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832695296:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764142.141319:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764142.141321:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764142.141322:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764142.141323:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764142.141326:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613000.
02000000:00000001:3.0:1787764142.141327:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764142.141330:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764142.141332:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787764142.141334:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787764142.141343:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832695296, portal 10
00000100:00000001:3.0:1787764142.141344:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764142.141346:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497721344 : -115097211830272 : ffff9751d7930000)
00000100:00000040:3.0:1787764142.141350:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930000 x1874599832695296/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764167 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141355:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764142.141357:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764142.141359:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787764142.141360:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832695296, offset 0
00000400:00000200:3.0:1787764142.141363:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764142.141366:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764142.141368:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764142.141370:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764142.141372:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764142.141374:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764142.141375:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764142.141377:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58667:58667:256:4294967295] 192.168.203.110@tcp LPNI seq info [58667:58667:8:4294967295]
00000400:00000200:3.0:1787764142.141380:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764142.141385:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764142.141387:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764142.141390:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787764142.141392:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764142.141395:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787764142.141398:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764142.141400:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1787764142.141402:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764142.141404:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787764142.141406:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764142.141408:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764142.141410:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787764142.141411:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787764142.141413:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764142.141415:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930000 x1874599832695296/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764167 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141421:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764142.141423:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930000 x1874599832695296/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764167 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141432:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764142.141433:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764142.141435:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764142.141438:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764142.141440:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764142.141441:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316529184
00000100:00000001:3.0:1787764142.141443:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316530419
00000100:00000001:3.0:1787764142.141445:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316529184
00000100:00000001:3.0:1787764142.141446:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316530419
00000100:00000001:3.0:1787764142.141448:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764142.141449:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764142.141450:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764142.141452:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764142/real 1787764142]  req@ffff9751d7930000 x1874599832695296/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764142.141457:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764142.141458:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764142.141459:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787764142.141461:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787764142.141462:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787764142.141463:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764142.141465:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930000 x1874599832695296/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764142.141470:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930000 x1874599832695296/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141474:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764142.141475:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764142.141476:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764142.141477:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764142.141479:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930000 x1874599832695296/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141484:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764142.141487:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930000 x1874599832695296/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141491:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764142.141493:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764142.141494:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764142.141494:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764142.141495:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764142.141505:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764142.141507:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764142.141508:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764142.141510:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764142.141512:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930000 x1874599832695296/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764142.141517:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832695296:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764142.141520:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764142.141522:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930b40 x1874599832695424/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141526:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764142.141527:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764142.141528:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764142.141529:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764142.141530:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764142.141532:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832695424:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764142.141534:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764142.141535:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764142.141536:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764142.141538:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764142.141540:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612800.
02000000:00000001:3.0:1787764142.141541:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764142.141543:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764142.141545:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787764142.141547:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787764142.141548:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832695424, portal 10
00000100:00000001:3.0:1787764142.141550:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764142.141551:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497724224 : -115097211827392 : ffff9751d7930b40)
00000100:00000040:3.0:1787764142.141554:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930b40 x1874599832695424/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764167 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141558:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764142.141560:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764142.141561:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787764142.141563:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832695424, offset 0
00000400:00000200:3.0:1787764142.141565:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764142.141567:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764142.141569:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764142.141571:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764142.141573:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764142.141574:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764142.141575:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764142.141578:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58668:58668:256:4294967295] 192.168.203.110@tcp LPNI seq info [58668:58668:8:4294967295]
00000400:00000200:3.0:1787764142.141580:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764142.141584:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764142.141587:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764142.141588:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787764142.141591:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764142.141592:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787764142.141595:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764142.141597:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1787764142.141599:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764142.141601:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787764142.141602:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764142.141604:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764142.141606:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787764142.141607:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787764142.141609:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764142.141611:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930b40 x1874599832695424/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764167 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141616:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764142.141618:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930b40 x1874599832695424/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764167 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141622:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764142.141623:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764142.141625:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764142.141626:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764142.141628:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764142.141629:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316529172
00000100:00000001:3.0:1787764142.141631:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316530419
00000100:00000001:3.0:1787764142.141632:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316529172
00000100:00000001:3.0:1787764142.141634:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316530419
00000100:00000001:3.0:1787764142.141637:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764142.141639:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764142.141640:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764142.141644:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764142/real 1787764142]  req@ffff9751d7930b40 x1874599832695424/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764142.141653:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764142.141654:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764142.141656:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787764142.141658:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787764142.141659:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787764142.141662:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764142.141666:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930b40 x1874599832695424/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764142.141675:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930b40 x1874599832695424/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141682:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764142.141684:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764142.141686:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764142.141688:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764142.141690:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930b40 x1874599832695424/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141695:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764142.141697:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930b40 x1874599832695424/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141702:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764142.141703:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764142.141704:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764142.141705:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764142.141706:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764142.141707:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764142.141709:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764142.141711:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764142.141712:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764142.141714:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930b40 x1874599832695424/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764142.141719:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832695424:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764142.141722:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764142.141724:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7931680 x1874599832695552/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141729:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764142.141730:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764142.141731:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764142.141732:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764142.141734:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764142.141735:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832695552:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764142.141738:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764142.141739:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764142.141740:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764142.141741:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764142.141744:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612c00.
02000000:00000001:3.0:1787764142.141745:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764142.141748:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764142.141749:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787764142.141752:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787764142.141754:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832695552, portal 4
00000100:00000001:3.0:1787764142.141755:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764142.141756:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497727104 : -115097211824512 : ffff9751d7931680)
00000100:00000040:3.0:1787764142.141760:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7931680 x1874599832695552/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764167 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141764:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764142.141766:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764142.141767:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787764142.141769:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832695552, offset 0
00000400:00000200:3.0:1787764142.141771:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764142.141773:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764142.141776:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764142.141777:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764142.141779:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764142.141781:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764142.141782:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764142.141784:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58669:58669:256:4294967295] 192.168.203.110@tcp LPNI seq info [58669:58669:8:4294967295]
00000400:00000200:3.0:1787764142.141787:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764142.141792:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764142.141796:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764142.141800:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787764142.141804:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764142.141807:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787764142.141811:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764142.141813:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1787764142.141819:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764142.141821:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787764142.141824:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764142.141827:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764142.141831:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787764142.141833:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787764142.141835:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764142.141839:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7931680 x1874599832695552/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764167 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141846:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764142.141848:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7931680 x1874599832695552/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764167 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141852:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764142.141853:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764142.141856:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764142.141858:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764142.141860:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764142.141861:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316529189
00000100:00000001:3.0:1787764142.141863:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316530419
00000100:00000001:3.0:1787764142.141865:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316529189
00000100:00000001:3.0:1787764142.141866:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316530419
00000100:00000001:3.0:1787764142.141868:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764142.141870:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764142.141871:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764142.141873:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764142/real 1787764142]  req@ffff9751d7931680 x1874599832695552/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764142.141879:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764142.141879:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764142.141880:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787764142.141882:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787764142.141883:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787764142.141884:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764142.141886:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7931680 x1874599832695552/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764142.141892:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7931680 x1874599832695552/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141895:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764142.141896:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764142.141897:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764142.141898:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764142.141900:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7931680 x1874599832695552/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141904:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764142.141906:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7931680 x1874599832695552/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141910:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764142.141912:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764142.141913:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764142.141914:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764142.141915:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764142.141916:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764142.141918:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764142.141921:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764142.141922:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764142.141925:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7931680 x1874599832695552/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764142.141929:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832695552:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764142.141932:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764142.141933:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764142.141935:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930000 x1874599832695296/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141939:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764142.141940:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764142.141941:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613000.
02000000:00000001:3.0:1787764142.141942:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764142.141943:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764142.141944:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764142.141946:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764142.141947:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:3.0:1787764142.141949:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764142.141950:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764142.141951:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930000.
00000100:00000001:3.0:1787764142.141952:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764142.141953:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764142.141954:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764142.141956:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930b40 x1874599832695424/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141960:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764142.141960:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764142.141961:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612800.
02000000:00000001:3.0:1787764142.141963:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764142.141963:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764142.141964:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764142.141965:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764142.141967:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cac00.
02000000:00000001:3.0:1787764142.141968:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764142.141969:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764142.141970:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930b40.
00000100:00000001:3.0:1787764142.141971:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764142.141972:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764142.141973:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764142.141974:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7931680 x1874599832695552/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764167 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764142.141978:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764142.141979:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764142.141980:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612c00.
02000000:00000001:3.0:1787764142.141981:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764142.141981:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764142.141982:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764142.141983:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764142.141984:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9400.
02000000:00000001:3.0:1787764142.141985:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764142.141986:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764142.141986:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7931680.
00000100:00000001:3.0:1787764142.141988:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764142.141988:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764142.141990:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764142.141992:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764142.141993:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764142.141994:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764142.141995:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764142.780272:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764142.780276:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764142.780283:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764142.780285:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764142.780289:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764142.780290:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764142.780293:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764142.780296:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764142.780298:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764142.780303:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764142.780303:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764142.780306:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764142.780310:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764142.780310:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764142.780316:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764142.780317:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764142.780321:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764142.780321:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764142.780327:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764142.780330:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764142.780331:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764142.780376:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764142.780395:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764142.780419:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764143.164224:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764143.164231:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764143.164232:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764143.164248:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764143.164253:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764143.164257:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764143.164259:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764143.164261:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764143.164263:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764143.420254:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316530419, at 4316530432
00000100:00000010:3.0:1787764143.420273:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40f00.
00000100:00000040:2.0:1787764143.420291:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316530419, at 4316530432
00000100:00000010:2.0:1787764143.420301:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d403c0.
00000100:00000001:2.0:1787764143.804198:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764143.804206:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764143.804208:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764143.804212:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764143.804212:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764143.804216:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764143.804218:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764143.804222:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764143.804222:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764143.804226:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764143.804227:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764143.804231:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764143.804231:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764143.804246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764143.804247:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764143.804325:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764143.804328:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764143.804331:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764144.188261:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764144.188270:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764144.188272:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764144.188278:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764144.188283:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764144.188287:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764144.188290:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764144.188293:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764144.188296:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764144.828248:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764144.828249:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764144.828256:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764144.828258:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764144.828262:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764144.828265:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764144.828267:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764144.828271:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764144.828273:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764144.828276:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764144.828277:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764144.828281:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764144.828282:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764144.828284:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764144.828286:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764144.828287:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764144.828292:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764144.828294:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764144.828298:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764144.828302:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764144.828304:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764144.828343:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764144.828347:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764144.828350:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764145.212254:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764145.212261:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764145.212263:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764145.212268:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764145.212272:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764145.212275:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764145.212277:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764145.212279:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764145.212282:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764145.852253:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764145.852261:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764145.852269:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764145.852270:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764145.852274:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764145.852276:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764145.852320:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764145.852325:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764145.852329:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764145.852389:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764145.852394:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764145.852397:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764145.852400:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764145.852405:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764145.852408:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764145.852411:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764145.852414:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764145.852417:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764146.236265:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764146.236279:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764146.236282:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764146.236288:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764146.236294:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764146.236298:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764146.236301:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764146.236304:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764146.236308:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764146.876255:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764146.876262:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764146.876269:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764146.876271:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764146.876275:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764146.876277:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764146.876349:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764146.876354:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764146.876359:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764146.876394:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764146.876399:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764146.876402:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764146.876406:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764146.876410:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764146.876414:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764146.876416:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764146.876419:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764146.876422:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764147.260254:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764147.260263:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1787764147.260266:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764147.260266:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764147.260274:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00100000:3.0:1787764147.260274:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787764147.260277:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1787764147.260280:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:0.0:1787764147.260281:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764147.260287:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764147.260288:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764147.260293:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:0.0:1787764147.260295:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21861
00000100:00000001:3.0:1787764147.260297:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.260301:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787764147.260305:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764147.260313:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764147.260316:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764147.260320:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764147.260324:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764147.260327:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764147.260330:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764147.260334:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764147.260336:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764147.260339:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764147.260342:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764147.260345:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764147.260348:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787764147.260351:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764147.260352:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764147.260354:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764147.260355:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764147.260358:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787764147.260361:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764147.260363:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764147.260365:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764147.260367:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764147.260369:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764147.260371:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764147.260373:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764147.260380:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787764147.260389:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764147.260392:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764147.260394:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764147.260396:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764147.260398:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764147.260401:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764147.260408:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787764147.260413:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764147.260420:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832695680/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764147.260428:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764147.260431:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764147.260434:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832695680/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764147.260450:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764147.260453:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764147.260456:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764147.260458:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764147.260460:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764147.260463:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764147.260466:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21861
00000400:00000200:0.0:1787764147.260470:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764147.260474:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764147.260476:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764147.260479:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764147.260482:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764147.260484:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764147.260486:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764147.260486:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1787764147.260488:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1787764147.260489:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000040:0.0:1787764147.260490:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787764147.260491:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787764147.260491:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1787764147.260493:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787764147.260496:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787764147.260498:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:3.0:1787764147.260498:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e940 x1874599832695680/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764147.260502:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764147.260504:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764147.260507:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764147.260507:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764147.260509:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764147.260509:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787764147.260512:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:3.0:1787764147.260513:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:0.0:1787764147.260514:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:3.0:1787764147.260515:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764147.260516:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764147.260518:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764147.260518:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764147.260522:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832695680:192.168.203.110@tcp:250:kworker.0
02000000:00000001:0.0:1787764147.260523:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764147.260526:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764147.260527:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1787764147.260528:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1787764147.260530:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00100000:0.0:1787764147.260532:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1787764147.260532:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764147.260535:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:0.0:1787764147.260539:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764147.260540:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612c00.
00000100:00000001:0.0:1787764147.260542:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764147.260542:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764147.260545:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000020:00000040:0.0:1787764147.260547:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000200:3.0:1787764147.260548:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787764147.260550:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764147.260551:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764147.260552:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000100:00000001:0.0:1787764147.260553:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:3.0:1787764147.260559:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
02000000:00000001:0.0:1787764147.260562:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764147.260563:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764147.260565:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832695680, portal 25
02000000:00000010:0.0:1787764147.260567:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:3.0:1787764147.260568:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764147.260570:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049528640 : -115096660022976 : ffff9751f876e940)
00000100:00000001:0.0:1787764147.260571:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764147.260576:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832695808/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:3.0:1787764147.260576:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e940 x1874599832695680/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764172 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764147.260585:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764147.260585:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1787764147.260588:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764147.260589:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764147.260591:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000040:0.0:1787764147.260592:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832695808/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764147.260594:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832695680, offset 0
00000100:00000001:0.0:1787764147.260601:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764147.260604:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764147.260608:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:3.0:1787764147.260608:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787764147.260610:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764147.260611:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764147.260614:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764147.260617:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21861
00000400:00000200:3.0:1787764147.260617:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764147.260620:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764147.260622:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1787764147.260623:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00080000:0.0:1787764147.260625:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764147.260628:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:3.0:1787764147.260629:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1787764147.260630:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764147.260632:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:3.0:1787764147.260633:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:0.0:1787764147.260634:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764147.260636:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:3.0:1787764147.260637:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000020:00000040:0.0:1787764147.260638:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787764147.260639:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:3.0:1787764147.260640:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000020:00000040:0.0:1787764147.260641:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787764147.260644:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:3.0:1787764147.260644:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58670:58670:256:4294967295] 192.168.203.110@tcp LPNI seq info [58670:58670:8:4294967295]
00000020:00000001:0.0:1787764147.260647:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:3.0:1787764147.260649:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:0.0:1787764147.260650:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764147.260652:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764147.260654:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764147.260656:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764147.260659:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:3.0:1787764147.260659:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00080000:0.0:1787764147.260662:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764147.260664:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787764147.260665:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
02000000:00000001:0.0:1787764147.260666:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764147.260668:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764147.260671:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1787764147.260671:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000100:00000001:0.0:1787764147.260673:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764147.260674:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787764147.260677:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787764147.260678:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787764147.260679:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764147.260683:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
00000800:00000200:3.0:1787764147.260683:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000020:00000040:0.0:1787764147.260685:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787764147.260687:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764147.260689:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787764147.260689:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787764147.260691:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764147.260692:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:3.1:1787764147.260693:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
02000000:00000001:0.0:1787764147.260694:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1787764147.260697:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
02000000:00000010:0.0:1787764147.260698:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000800:00000010:3.0:1787764147.260700:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000100:00000001:0.0:1787764147.260701:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764147.260705:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832695936/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787764147.260706:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764147.260711:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764147.260716:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787764147.260719:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:0.0:1787764147.260720:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764147.260722:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764147.260723:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:0.0:1787764147.260726:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832695936/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764147.260727:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e940 x1874599832695680/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764172 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764147.260735:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764147.260738:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787764147.260742:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1787764147.260743:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00080000:0.0:1787764147.260745:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1787764147.260747:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e940 x1874599832695680/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764172 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787764147.260749:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787764147.260750:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764147.260752:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764147.260755:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764147.260755:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764147.260757:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21861
00000100:00000001:3.0:1787764147.260757:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000400:00000200:0.0:1787764147.260761:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764147.260764:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787764147.260764:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764147.260766:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1787764147.260768:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:0.0:1787764147.260769:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764147.260771:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316535539
00000100:00000001:0.0:1787764147.260772:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764147.260774:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764147.260776:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000010:3.0:1787764147.260776:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40f00.
00000020:00000001:0.0:1787764147.260779:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764147.260780:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787764147.260781:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764147.260783:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000001:3.0:1787764147.260784:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000020:00000001:0.0:1787764147.260791:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:3.0:1787764147.260791:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316535539
00000020:00000001:0.0:1787764147.260794:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787764147.260797:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764147.260799:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000010:3.0:1787764147.260799:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d407e0.
00000100:00000040:0.0:1787764147.260801:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764147.260803:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787764147.260804:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764147.260806:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:3.0:1787764147.260806:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:0.0:1787764147.260809:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764147.260809:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764147.260812:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764147.260812:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000001:0.0:1787764147.260814:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764147.260816:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:3.0:1787764147.260816:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764147/real 1787764147]  req@ffff9751f876e940 x1874599832695680/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787764147.260819:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764147.260821:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764147.260823:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000400:00000001:3.0:1787764147.260825:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000008:00000001:0.0:1787764147.260826:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1787764147.260827:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:0.0:1787764147.260828:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764147.260830:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000100:00000010:0.0:1787764147.260832:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787764147.260834:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787764147.260837:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764147.260838:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764147.260838:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000100:00000001:0.0:1787764147.260840:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:3.0:1787764147.260840:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
02000000:00000001:0.0:1787764147.260842:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1787764147.260843:0:257575:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:0.0:1787764147.260844:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764147.260846:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e940 x1874599832695680/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1787764147.260847:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787764147.260850:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764147.260854:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832696064/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1787764147.260856:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e940 x1874599832695680/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764147.260863:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764147.260864:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787764147.260865:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764147.260866:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.260868:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000040:0.0:1787764147.260869:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832696064/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.260870:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764147.260875:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e940 x1874599832695680/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764147.260877:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764147.260881:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21871)
00000100:00000001:3.0:1787764147.260882:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764147.260887:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e940 x1874599832695680/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.260895:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764147.260898:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.260900:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764147.260902:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.260904:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764147.260906:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764147.260909:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764147.260913:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764147.260916:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764147.260920:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e940 x1874599832695680/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764147.260926:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832695680:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764147.260929:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.260931:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764147.260933:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e940 x1874599832695680/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.260937:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764147.260939:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764147.260941:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612c00.
02000000:00000001:3.0:1787764147.260949:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764147.260951:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764147.260953:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764147.260955:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764147.260958:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014400.
02000000:00000001:3.0:1787764147.260961:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764147.260963:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764147.260965:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e940.
00000100:00000001:3.0:1787764147.260969:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764147.260971:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.260973:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.260980:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764147.260981:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764147.260984:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764147.260986:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764147.260987:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764147.260992:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cf00 x1874599832695808/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261000:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764147.261001:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764147.261004:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764147.261005:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764147.261006:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764147.261009:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832695808:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764147.261011:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764147.261012:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764147.261014:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764147.261015:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764147.261017:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612c00.
02000000:00000001:3.0:1787764147.261019:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764147.261021:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764147.261023:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787764147.261026:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787764147.261028:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832695808, portal 10
00000100:00000001:3.0:1787764147.261030:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764147.261031:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049521920 : -115096660029696 : ffff9751f876cf00)
00000100:00000040:3.0:1787764147.261036:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cf00 x1874599832695808/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764172 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261040:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764147.261042:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764147.261043:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787764147.261045:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832695808, offset 0
00000400:00000200:3.0:1787764147.261047:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764147.261050:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764147.261053:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764147.261055:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764147.261057:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764147.261059:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764147.261060:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764147.261063:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58671:58671:256:4294967295] 192.168.203.110@tcp LPNI seq info [58671:58671:8:4294967295]
00000400:00000200:3.0:1787764147.261066:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764147.261070:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764147.261073:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764147.261075:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787764147.261078:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764147.261080:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787764147.261083:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764147.261085:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1787764147.261088:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764147.261089:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787764147.261091:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764147.261094:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764147.261096:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787764147.261097:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787764147.261099:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764147.261101:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cf00 x1874599832695808/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764172 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261107:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764147.261109:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cf00 x1874599832695808/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764172 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261114:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764147.261115:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764147.261117:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764147.261119:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764147.261121:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764147.261123:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316534303
00000100:00000001:3.0:1787764147.261124:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316535539
00000100:00000001:3.0:1787764147.261126:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316534303
00000100:00000001:3.0:1787764147.261128:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316535539
00000100:00000001:3.0:1787764147.261130:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764147.261131:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764147.261149:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764147.261152:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764147/real 1787764147]  req@ffff9751f876cf00 x1874599832695808/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764147.261158:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764147.261159:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764147.261160:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787764147.261162:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787764147.261163:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787764147.261164:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764147.261166:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cf00 x1874599832695808/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764147.261172:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cf00 x1874599832695808/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261176:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764147.261177:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.261178:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764147.261179:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764147.261181:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cf00 x1874599832695808/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261185:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764147.261188:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cf00 x1874599832695808/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261192:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764147.261194:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.261195:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764147.261196:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.261197:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764147.261206:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764147.261228:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764147.261231:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764147.261232:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764147.261235:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cf00 x1874599832695808/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764147.261241:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832695808:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764147.261244:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764147.261246:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f840 x1874599832695936/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261250:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764147.261251:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764147.261253:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764147.261254:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764147.261255:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764147.261257:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832695936:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764147.261259:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764147.261261:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764147.261262:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764147.261263:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764147.261266:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612800.
02000000:00000001:3.0:1787764147.261267:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764147.261269:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764147.261271:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787764147.261273:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787764147.261275:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832695936, portal 10
00000100:00000001:3.0:1787764147.261277:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764147.261278:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049532480 : -115096660019136 : ffff9751f876f840)
00000100:00000040:3.0:1787764147.261282:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f840 x1874599832695936/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764172 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261290:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764147.261292:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764147.261293:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787764147.261295:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832695936, offset 0
00000400:00000200:3.0:1787764147.261297:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764147.261300:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764147.261309:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764147.261311:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764147.261313:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764147.261315:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764147.261316:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764147.261319:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58672:58672:256:4294967295] 192.168.203.110@tcp LPNI seq info [58672:58672:8:4294967295]
00000400:00000200:3.0:1787764147.261322:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764147.261326:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764147.261330:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764147.261332:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787764147.261335:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764147.261337:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787764147.261340:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764147.261341:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1787764147.261344:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764147.261346:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787764147.261348:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764147.261350:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764147.261352:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787764147.261354:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787764147.261355:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764147.261358:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f840 x1874599832695936/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764172 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261363:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764147.261365:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f840 x1874599832695936/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764172 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261374:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764147.261375:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764147.261377:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764147.261379:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764147.261381:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764147.261383:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316534292
00000100:00000001:3.0:1787764147.261384:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316535539
00000100:00000001:3.0:1787764147.261386:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316534292
00000100:00000001:3.0:1787764147.261387:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316535539
00000100:00000001:3.0:1787764147.261389:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764147.261390:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764147.261391:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764147.261393:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764147/real 1787764147]  req@ffff9751f876f840 x1874599832695936/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764147.261398:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764147.261398:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764147.261400:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787764147.261401:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787764147.261402:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787764147.261403:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764147.261405:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f840 x1874599832695936/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764147.261410:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f840 x1874599832695936/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261414:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764147.261415:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.261416:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764147.261417:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764147.261419:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f840 x1874599832695936/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261424:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764147.261426:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f840 x1874599832695936/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261431:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764147.261432:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.261433:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764147.261434:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.261435:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764147.261436:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764147.261437:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764147.261439:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764147.261440:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764147.261443:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f840 x1874599832695936/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764147.261448:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832695936:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764147.261450:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764147.261452:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832696064/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261455:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764147.261456:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764147.261457:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764147.261459:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764147.261460:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764147.261462:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832696064:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764147.261464:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764147.261466:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764147.261467:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764147.261468:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764147.261470:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613000.
02000000:00000001:3.0:1787764147.261472:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764147.261474:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764147.261475:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787764147.261477:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787764147.261479:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832696064, portal 4
00000100:00000001:3.0:1787764147.261480:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764147.261481:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
00000100:00000040:3.0:1787764147.261485:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832696064/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764172 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261489:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764147.261491:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764147.261492:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787764147.261493:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832696064, offset 0
00000400:00000200:3.0:1787764147.261495:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764147.261497:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764147.261499:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764147.261501:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764147.261503:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764147.261504:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764147.261506:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764147.261508:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58673:58673:256:4294967295] 192.168.203.110@tcp LPNI seq info [58673:58673:8:4294967295]
00000400:00000200:3.0:1787764147.261510:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764147.261514:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764147.261517:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764147.261519:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787764147.261522:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764147.261523:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787764147.261526:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764147.261527:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1787764147.261530:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764147.261532:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787764147.261533:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764147.261535:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764147.261537:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787764147.261538:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787764147.261540:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764147.261542:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876ed00 x1874599832696064/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764172 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261546:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764147.261548:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832696064/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764172 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261552:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764147.261553:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764147.261555:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764147.261556:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764147.261558:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764147.261560:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316534309
00000100:00000001:3.0:1787764147.261561:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316535539
00000100:00000001:3.0:1787764147.261563:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316534309
00000100:00000001:3.0:1787764147.261564:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316535539
00000100:00000001:3.0:1787764147.261565:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764147.261567:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764147.261568:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764147.261570:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764147/real 1787764147]  req@ffff9751f876ed00 x1874599832696064/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764147.261574:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764147.261575:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764147.261576:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787764147.261577:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787764147.261578:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787764147.261580:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764147.261582:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832696064/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764147.261587:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832696064/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261591:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764147.261592:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.261593:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764147.261594:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764147.261596:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832696064/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261599:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764147.261602:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832696064/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261606:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764147.261607:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.261608:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764147.261609:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.261610:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764147.261611:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764147.261612:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764147.261614:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764147.261615:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764147.261617:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832696064/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764147.261622:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832696064:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764147.261624:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.261625:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764147.261628:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cf00 x1874599832695808/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261632:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764147.261633:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764147.261635:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612c00.
02000000:00000001:3.0:1787764147.261636:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764147.261637:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764147.261638:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764147.261639:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764147.261640:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
02000000:00000001:3.0:1787764147.261642:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764147.261643:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764147.261644:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cf00.
00000100:00000001:3.0:1787764147.261647:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764147.261648:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.261649:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764147.261651:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f840 x1874599832695936/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261655:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764147.261656:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764147.261657:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612800.
02000000:00000001:3.0:1787764147.261658:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764147.261658:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764147.261659:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764147.261660:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764147.261661:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:3.0:1787764147.261662:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764147.261663:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764147.261664:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f840.
00000100:00000001:3.0:1787764147.261665:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764147.261666:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.261667:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764147.261669:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832696064/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764172 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764147.261672:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764147.261674:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764147.261675:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613000.
02000000:00000001:3.0:1787764147.261676:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764147.261677:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764147.261678:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764147.261679:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764147.261680:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017800.
02000000:00000001:3.0:1787764147.261682:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764147.261683:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764147.261684:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:3.0:1787764147.261685:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764147.261686:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.261687:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764147.261689:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764147.261690:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764147.261691:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764147.261692:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764147.900236:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764147.900244:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764147.900247:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764147.900250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764147.900253:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764147.900255:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764147.900259:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764147.900263:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764147.900264:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764147.900269:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764147.900270:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764147.900272:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764147.900274:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764147.900277:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764147.900279:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764147.900285:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764147.900287:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764147.900289:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764147.900292:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764147.900297:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764147.900298:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764147.900350:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764147.900354:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764147.900357:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764148.284252:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764148.284259:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764148.284260:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764148.284265:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764148.284269:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764148.284272:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764148.284274:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764148.284277:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764148.284280:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764148.540223:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316535539, at 4316535552
00000100:00000010:3.0:1787764148.540236:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d407e0.
00000100:00000040:2.0:1787764148.540284:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316535539, at 4316535552
00000100:00000010:2.0:1787764148.540300:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40f00.
00010000:00000001:3.0:1787764148.924243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764148.924249:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764148.924255:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764148.924257:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764148.924261:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764148.924263:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764148.924318:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764148.924322:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764148.924324:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764148.924326:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764148.924329:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764148.924331:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764148.924333:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764148.924334:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764148.924336:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764148.924369:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764148.924373:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764148.924376:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764149.308211:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764149.308217:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764149.308219:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764149.308223:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764149.308230:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764149.308233:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764149.308234:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764149.308238:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764149.308240:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764149.948254:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764149.948258:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764149.948262:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764149.948264:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764149.948266:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764149.948273:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764149.948273:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764149.948278:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764149.948282:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764149.948285:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764149.948286:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764149.948295:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764149.948295:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764149.948298:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764149.948300:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764149.948345:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764149.948348:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764149.948351:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764150.332227:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764150.332235:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764150.332237:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764150.332241:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764150.332244:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764150.332247:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764150.332249:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764150.332250:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764150.332252:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764150.972237:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764150.972240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764150.972244:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764150.972244:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764150.972249:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764150.972250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764150.972253:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764150.972255:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764150.972256:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764150.972259:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764150.972260:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764150.972263:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764150.972266:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764150.972268:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764150.972271:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764150.972301:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764150.972306:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764150.972309:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764151.356249:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764151.356256:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764151.356258:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764151.356263:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764151.356266:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764151.356269:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764151.356272:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764151.356274:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764151.356277:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764151.996249:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764151.996258:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764151.996260:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764151.996265:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764151.996266:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764151.996269:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764151.996269:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764151.996271:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764151.996273:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764151.996274:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764151.996275:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764151.996278:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764151.996281:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764151.996285:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764151.996287:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764151.996322:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764151.996326:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764151.996329:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787764152.380258:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764152.380260:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:0.0:1787764152.380268:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764152.380268:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764152.380272:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764152.380273:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764152.380279:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:3.0:1787764152.380279:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787764152.380286:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764152.380286:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764152.380291:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787764152.380295:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21866
00000100:00000001:3.0:1787764152.380295:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764152.380299:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.380304:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787764152.380309:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764152.380316:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764152.380319:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764152.380322:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764152.380325:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764152.380327:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764152.380330:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764152.380334:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764152.380336:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764152.380339:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764152.380342:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764152.380345:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764152.380347:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787764152.380350:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764152.380352:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764152.380354:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764152.380356:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764152.380359:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787764152.380362:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764152.380364:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764152.380367:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764152.380369:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764152.380372:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764152.380375:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764152.380377:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764152.380386:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787764152.380389:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764152.380393:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764152.380394:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764152.380397:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764152.380399:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764152.380403:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764152.380409:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787764152.380416:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764152.380424:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832696192/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764152.380434:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764152.380438:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764152.380442:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832696192/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764152.380467:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764152.380470:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764152.380473:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764152.380475:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764152.380476:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764152.380479:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764152.380482:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21866
00000400:00000200:0.0:1787764152.380485:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764152.380489:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764152.380492:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764152.380495:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764152.380497:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764152.380499:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1787764152.380500:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764152.380504:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764152.380505:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764152.380508:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1787764152.380509:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764152.380512:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00000040:3.0:1787764152.380517:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f0c0 x1874599832696192/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787764152.380521:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764152.380525:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000001:3.0:1787764152.380530:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:0.0:1787764152.380532:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:3.0:1787764152.380533:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764152.380535:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:3.0:1787764152.380538:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:0.0:1787764152.380541:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:3.0:1787764152.380543:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764152.380545:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764152.380549:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:3.0:1787764152.380550:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764152.380553:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764152.380558:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00100000:3.0:1787764152.380559:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832696192:192.168.203.110@tcp:250:kworker.0
00000100:00080000:0.0:1787764152.380564:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764152.380569:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764152.380573:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764152.380573:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764152.380579:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787764152.380581:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764152.380582:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764152.380585:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764152.380586:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1787764152.380589:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000010:3.0:1787764152.380593:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613000.
00000008:00100000:0.0:1787764152.380594:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1787764152.380597:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1787764152.380599:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764152.380602:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764152.380606:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000010:0.0:1787764152.380609:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000400:00000010:3.0:1787764152.380612:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000020:00000040:0.0:1787764152.380613:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787764152.380618:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764152.380620:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764152.380622:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:3.0:1787764152.380622:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
02000000:00000001:0.0:1787764152.380624:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764152.380628:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764152.380632:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832696192, portal 25
02000000:00000010:0.0:1787764152.380633:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:3.0:1787764152.380637:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787764152.380639:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764152.380641:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049530560 : -115096660021056 : ffff9751f876f0c0)
00000100:00100000:0.0:1787764152.380647:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832696320/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:3.0:1787764152.380651:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f0c0 x1874599832696192/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764177 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764152.380666:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764152.380670:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1787764152.380671:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764152.380675:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787764152.380679:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832696320/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1787764152.380681:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787764152.380687:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832696192, offset 0
00000100:00000001:0.0:1787764152.380694:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764152.380697:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1787764152.380699:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00080000:0.0:1787764152.380702:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787764152.380705:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764152.380706:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764152.380710:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1787764152.380711:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:0.0:1787764152.380716:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21866
00000400:00000200:3.0:1787764152.380720:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:0.0:1787764152.380723:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764152.380728:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1787764152.380730:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764152.380733:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1787764152.380734:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000040:0.0:1787764152.380740:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:3.0:1787764152.380741:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1787764152.380745:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787764152.380746:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000040:0.0:1787764152.380751:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:3.0:1787764152.380756:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58674:58674:256:4294967295] 192.168.203.110@tcp LPNI seq info [58674:58674:8:4294967295]
00000100:00000001:0.0:1787764152.380757:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764152.380762:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764152.380765:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000400:00000200:3.0:1787764152.380766:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:0.0:1787764152.380769:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764152.380773:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787764152.380778:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787764152.380783:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:3.0:1787764152.380783:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:0.0:1787764152.380791:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:3.0:1787764152.380795:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787764152.380796:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764152.380801:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000010:3.0:1787764152.380803:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000100:00000001:0.0:1787764152.380804:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764152.380809:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787764152.380814:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000800:00000200:3.0:1787764152.380816:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787764152.380819:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764152.380822:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:3.0:1787764152.380822:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
02000000:00000001:0.0:1787764152.380826:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764152.380830:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787764152.380831:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787764152.380833:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764152.380837:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000800:00000200:3.1:1787764152.380838:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000008:00000001:0.0:1787764152.380842:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764152.380847:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1787764152.380847:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764152.380851:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000100:00000010:0.0:1787764152.380854:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787764152.380858:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000400:00000200:3.0:1787764152.380858:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:0.0:1787764152.380861:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764152.380864:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764152.380867:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1787764152.380868:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764152.380871:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764152.380874:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764152.380875:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
02000000:00000010:0.0:1787764152.380878:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000400:00000010:3.0:1787764152.380879:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:0.0:1787764152.380883:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764152.380885:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00100000:0.0:1787764152.380891:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832696448/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1787764152.380893:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f0c0 x1874599832696192/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764177 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764152.380913:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764152.380917:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764152.380917:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764152.380923:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f0c0 x1874599832696192/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764177 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764152.380925:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832696448/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.380946:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764152.380947:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764152.380950:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:0.0:1787764152.380952:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787764152.380957:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1787764152.380958:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764152.380959:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764152.380964:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000200:3.0:1787764152.380965:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764152.380967:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1787764152.380970:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316540659
00000100:00080000:0.0:1787764152.380971:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764152.380976:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000010:3.0:1787764152.380978:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d407e0.
00000100:00080000:0.0:1787764152.380980:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21866
00000400:00000200:0.0:1787764152.380986:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764152.380987:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000400:00000200:0.0:1787764152.380991:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764152.380994:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787764152.380997:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316540659
00000100:00000040:0.0:1787764152.380998:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764152.381002:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000010:3.0:1787764152.381003:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40420.
00000100:00000040:0.0:1787764152.381006:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764152.381009:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764152.381009:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000020:00000001:0.0:1787764152.381014:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1787764152.381015:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000040:0.0:1787764152.381016:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787764152.381020:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787764152.381021:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787764152.381023:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000001:3.0:1787764152.381027:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000020:00000001:0.0:1787764152.381028:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787764152.381031:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00100000:3.0:1787764152.381035:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764152/real 1787764152]  req@ffff9751f876f0c0 x1874599832696192/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764152.381037:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764152.381042:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764152.381047:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764152.381051:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000001:3.0:1787764152.381054:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000040:0.0:1787764152.381055:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000400:00000001:3.0:1787764152.381057:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00080000:0.0:1787764152.381060:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000010:3.0:1787764152.381062:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000100:00000001:0.0:1787764152.381064:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764152.381066:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
02000000:00000001:0.0:1787764152.381067:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:3.0:1787764152.381072:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
02000000:00000001:0.0:1787764152.381073:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764152.381077:0:257575:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:0.0:1787764152.381078:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764152.381082:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000200:3.0:1787764152.381085:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f0c0 x1874599832696192/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1787764152.381086:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787764152.381092:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764152.381096:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764152.381102:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000100:00000200:3.0:1787764152.381106:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f0c0 x1874599832696192/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787764152.381107:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787764152.381112:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764152.381115:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764152.381119:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764152.381122:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1787764152.381122:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000001:0.0:1787764152.381126:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764152.381126:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.381130:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000010:0.0:1787764152.381132:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:3.0:1787764152.381133:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764152.381155:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764152.381161:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832696576/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:3.0:1787764152.381169:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f0c0 x1874599832696192/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764152.381173:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764152.381190:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764152.381194:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787764152.381199:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832696576/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787764152.381204:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f0c0 x1874599832696192/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764152.381218:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764152.381222:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:0.0:1787764152.381225:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21877)
00000100:00000001:3.0:1787764152.381229:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.381232:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764152.381233:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.381236:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764152.381238:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764152.381241:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764152.381245:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764152.381248:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764152.381252:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f0c0 x1874599832696192/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764152.381260:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832696192:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764152.381267:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.381269:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764152.381272:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f0c0 x1874599832696192/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.381278:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764152.381280:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764152.381283:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613000.
02000000:00000001:3.0:1787764152.381286:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764152.381288:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764152.381289:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764152.381292:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764152.381294:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:3.0:1787764152.381299:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764152.381300:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764152.381302:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f0c0.
00000100:00000001:3.0:1787764152.381305:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764152.381306:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.381308:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.381313:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764152.381315:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764152.381317:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764152.381318:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764152.381320:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764152.381324:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832696320/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.381330:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764152.381332:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764152.381333:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764152.381335:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764152.381337:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764152.381340:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832696320:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764152.381343:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764152.381345:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764152.381346:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764152.381349:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764152.381351:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613000.
02000000:00000001:3.0:1787764152.381353:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764152.381357:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764152.381359:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787764152.381363:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787764152.381366:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832696320, portal 10
00000100:00000001:3.0:1787764152.381368:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764152.381369:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00000040:3.0:1787764152.381376:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832696320/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764177 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.381382:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764152.381384:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764152.381386:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787764152.381389:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832696320, offset 0
00000400:00000200:3.0:1787764152.381392:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764152.381396:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764152.381399:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764152.381402:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764152.381404:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764152.381407:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764152.381409:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764152.381412:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58675:58675:256:4294967295] 192.168.203.110@tcp LPNI seq info [58675:58675:8:4294967295]
00000400:00000200:3.0:1787764152.381416:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764152.381421:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764152.381426:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764152.381429:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787764152.381433:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764152.381435:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787764152.381440:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764152.381442:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:3.0:1787764152.381446:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764152.381448:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787764152.381451:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764152.381454:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764152.381457:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787764152.381459:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787764152.381461:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764152.381464:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e580 x1874599832696320/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764177 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.381472:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764152.381475:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832696320/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764177 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.381481:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764152.381483:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764152.381485:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764152.381488:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764152.381491:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764152.381493:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316539424
00000100:00000001:3.0:1787764152.381496:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316540659
00000100:00000001:3.0:1787764152.381498:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316539424
00000100:00000001:3.0:1787764152.381500:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316540659
00000100:00000001:3.0:1787764152.381502:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764152.381504:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764152.381505:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764152.381509:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764152/real 1787764152]  req@ffff9751f876e580 x1874599832696320/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764152.381517:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764152.381519:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764152.381521:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787764152.381523:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787764152.381524:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787764152.381527:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764152.381530:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832696320/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764152.381538:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832696320/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.381545:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764152.381546:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.381547:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764152.381549:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764152.381552:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832696320/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.381558:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764152.381561:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832696320/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.381568:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764152.381571:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.381572:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764152.381574:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.381575:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764152.381588:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764152.381590:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764152.381594:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764152.381596:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764152.381599:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832696320/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764152.381608:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832696320:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764152.381611:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764152.381614:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832696448/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.381626:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764152.381627:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764152.381629:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764152.381630:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764152.381632:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764152.381635:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832696448:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764152.381638:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764152.381640:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764152.381641:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764152.381643:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764152.381646:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612800.
02000000:00000001:3.0:1787764152.381648:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764152.381651:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764152.381653:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787764152.381656:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787764152.381659:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832696448, portal 10
00000100:00000001:3.0:1787764152.381661:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764152.381662:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000100:00000040:3.0:1787764152.381667:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832696448/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764177 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.381674:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764152.381676:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764152.381678:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787764152.381680:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832696448, offset 0
00000400:00000200:3.0:1787764152.381683:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764152.381686:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764152.381690:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764152.381692:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764152.381695:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764152.381697:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764152.381699:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764152.381702:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58676:58676:256:4294967295] 192.168.203.110@tcp LPNI seq info [58676:58676:8:4294967295]
00000400:00000200:3.0:1787764152.381706:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764152.381711:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764152.381716:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764152.381719:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787764152.381723:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764152.381725:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787764152.381729:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764152.381731:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:3.0:1787764152.381735:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764152.381737:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787764152.381740:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764152.381743:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764152.381746:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787764152.381747:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787764152.381749:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764152.381753:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e1c0 x1874599832696448/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764177 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.381759:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764152.381762:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832696448/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764177 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.381768:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764152.381769:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764152.381772:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764152.381775:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764152.381777:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764152.381779:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316539412
00000100:00000001:3.0:1787764152.381782:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316540659
00000100:00000001:3.0:1787764152.381783:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316539412
00000100:00000001:3.0:1787764152.381785:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316540659
00000100:00000001:3.0:1787764152.381787:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764152.381790:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764152.381791:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764152.381794:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764152/real 1787764152]  req@ffff9751f876e1c0 x1874599832696448/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764152.381802:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764152.381803:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764152.381805:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787764152.381807:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787764152.381808:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787764152.381810:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764152.381813:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832696448/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764152.381821:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832696448/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.381826:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764152.381828:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.381829:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764152.381830:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764152.381834:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832696448/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.381841:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764152.381844:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832696448/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.381850:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764152.381853:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.381854:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764152.381855:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.381857:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764152.381858:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764152.381860:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764152.381865:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764152.381867:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764152.381870:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832696448/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764152.381879:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832696448:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764152.381882:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764152.381885:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832696576/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.381891:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764152.381893:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764152.381895:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764152.381897:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764152.381899:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764152.381901:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832696576:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764152.381904:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764152.381906:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764152.381907:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764152.381909:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764152.381912:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612c00.
02000000:00000001:3.0:1787764152.381914:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764152.381917:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764152.381919:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000400:00000010:3.0:1787764152.381922:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000200:3.0:1787764152.381924:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832696576, portal 4
00000100:00000001:3.0:1787764152.381926:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764152.381927:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
00000100:00000040:3.0:1787764152.381932:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832696576/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764177 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.381938:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764152.381941:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764152.381943:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787764152.381945:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832696576, offset 0
00000400:00000200:3.0:1787764152.381949:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764152.381953:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764152.381956:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764152.381958:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764152.381961:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764152.381963:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764152.381965:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764152.381968:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58677:58677:256:4294967295] 192.168.203.110@tcp LPNI seq info [58677:58677:8:4294967295]
00000400:00000200:3.0:1787764152.381972:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764152.381977:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764152.381981:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764152.381984:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787764152.381988:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764152.381990:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787764152.381993:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764152.381996:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:3.0:1787764152.381999:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764152.382001:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787764152.382004:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764152.382007:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764152.382010:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787764152.382011:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787764152.382014:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764152.382018:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876c000 x1874599832696576/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764177 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.382025:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764152.382028:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832696576/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764177 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.382035:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764152.382037:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764152.382040:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764152.382043:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764152.382045:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764152.382047:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316539429
00000100:00000001:3.0:1787764152.382049:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316540659
00000100:00000001:3.0:1787764152.382051:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316539429
00000100:00000001:3.0:1787764152.382053:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316540659
00000100:00000001:3.0:1787764152.382055:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764152.382058:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764152.382059:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764152.382062:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764152/real 1787764152]  req@ffff9751f876c000 x1874599832696576/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764152.382069:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764152.382070:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764152.382072:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:3.0:1787764152.382074:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:3.0:1787764152.382075:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:3.0:1787764152.382077:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764152.382080:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832696576/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764152.382088:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832696576/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.382095:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764152.382096:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.382098:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764152.382099:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764152.382103:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832696576/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.382109:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764152.382112:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832696576/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.382120:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764152.382122:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.382123:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764152.382125:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.382126:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764152.382127:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764152.382130:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764152.382132:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764152.382149:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764152.382154:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832696576/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764152.382161:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832696576:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764152.382165:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.382167:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764152.382170:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832696320/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.382188:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764152.382189:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764152.382191:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613000.
02000000:00000001:3.0:1787764152.382193:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764152.382195:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764152.382196:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764152.382198:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764152.382200:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
02000000:00000001:3.0:1787764152.382203:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764152.382204:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764152.382205:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:3.0:1787764152.382209:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764152.382210:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.382212:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764152.382215:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832696448/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.382221:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764152.382222:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764152.382223:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612800.
02000000:00000001:3.0:1787764152.382226:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764152.382227:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764152.382228:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764152.382230:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764152.382231:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014400.
02000000:00000001:3.0:1787764152.382233:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764152.382234:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764152.382235:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:3.0:1787764152.382237:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764152.382239:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.382240:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764152.382244:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832696576/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764177 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764152.382250:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764152.382251:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764152.382252:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612c00.
02000000:00000001:3.0:1787764152.382254:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764152.382255:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764152.382256:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764152.382258:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764152.382259:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017000.
02000000:00000001:3.0:1787764152.382262:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764152.382263:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764152.382264:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:3.0:1787764152.382267:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764152.382268:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.382270:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764152.382273:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764152.382274:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764152.382276:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764152.382279:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764153.020227:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764153.020235:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764153.020237:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764153.020243:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764153.020247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764153.020251:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764153.020252:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764153.020256:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764153.020258:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764153.020260:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764153.020261:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764153.020265:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764153.020267:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764153.020269:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764153.020271:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764153.020275:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764153.020278:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764153.020281:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764153.020283:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764153.020285:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764153.020288:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764153.020339:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764153.020343:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764153.020346:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764153.404226:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764153.404234:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764153.404236:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764153.404241:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764153.404245:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764153.404248:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764153.404250:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764153.404253:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764153.404255:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764153.660219:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316540659, at 4316540672
00000100:00000010:3.0:1787764153.660232:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40420.
00000100:00000040:2.0:1787764153.660265:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316540659, at 4316540672
00000100:00000010:2.0:1787764153.660273:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d407e0.
00000100:00000001:2.0:1787764154.044242:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764154.044242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764154.044249:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764154.044250:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764154.044253:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764154.044256:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764154.044257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764154.044261:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764154.044261:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764154.044265:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764154.044267:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764154.044268:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764154.044270:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764154.044272:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764154.044274:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764154.044310:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764154.044314:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764154.044317:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764154.428249:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764154.428256:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764154.428258:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764154.428264:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764154.428269:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764154.428272:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764154.428275:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764154.428277:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764154.428280:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764155.068269:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764155.068278:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764155.068279:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764155.068283:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764155.068287:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00100000:2.0:1787764155.068290:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764155.068292:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764155.068296:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764155.068299:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764155.068301:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764155.068304:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764155.068307:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764155.068311:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764155.068315:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764155.068321:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764155.068324:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764155.068329:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764155.068336:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764155.068339:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764155.068345:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764155.068347:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764155.068390:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764155.068396:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764155.068399:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764155.452253:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764155.452259:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764155.452261:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764155.452265:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764155.452267:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764155.452270:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764155.452272:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764155.452274:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764155.452276:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764156.092253:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764156.092253:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764156.092260:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764156.092261:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764156.092264:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764156.092268:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764156.092269:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764156.092272:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764156.092274:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764156.092277:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764156.092279:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764156.092280:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764156.092283:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764156.092287:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764156.092290:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764156.092315:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764156.092319:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764156.092322:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764156.476238:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764156.476253:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764156.476255:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764156.476260:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764156.476265:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764156.476268:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764156.476270:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764156.476272:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764156.476275:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764157.116204:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764157.116205:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764157.116210:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764157.116211:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764157.116214:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764157.116215:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764157.116218:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764157.116220:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764157.116221:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764157.116225:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764157.116226:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764157.116229:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764157.116231:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764157.116233:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764157.116234:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764157.116257:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764157.116261:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764157.116263:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764157.500202:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764157.500215:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764157.500218:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764157.500225:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764157.500230:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764157.500233:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764157.500236:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764157.500239:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764157.500242:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764157.500250:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764157.500255:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787764157.500257:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764157.500260:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764157.500264:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764157.500270:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21872
00000400:00000200:2.0:1787764157.500276:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764157.500286:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764157.500290:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764157.500292:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764157.500294:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764157.500296:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764157.500297:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764157.500300:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764157.500301:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787764157.500303:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764157.500305:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787764157.500307:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787764157.500308:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787764157.500310:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764157.500312:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764157.500313:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764157.500314:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764157.500316:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787764157.500319:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764157.500320:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764157.500322:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764157.500324:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764157.500326:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764157.500327:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764157.500329:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764157.500336:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000020:00000040:2.0:1787764157.500338:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787764157.500340:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764157.500341:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764157.500343:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764157.500345:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764157.500348:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764157.500354:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bb800.
00000100:00000001:2.0:1787764157.500359:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764157.500365:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832696704/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764157.500372:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764157.500375:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764157.500378:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832696704/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764157.500391:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764157.500393:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764157.500395:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787764157.500396:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764157.500397:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764157.500398:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764157.500401:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21872
00000400:00000200:2.0:1787764157.500404:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764157.500406:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764157.500408:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764157.500410:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764157.500412:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764157.500414:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764157.500414:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764157.500418:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764157.500419:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1787764157.500420:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764157.500424:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00000001:3.0:1787764157.500424:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1787764157.500427:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764157.500429:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:2.0:1787764157.500431:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:2.0:1787764157.500433:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:3.0:1787764157.500433:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832696704/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787764157.500439:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764157.500441:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764157.500445:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764157.500448:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764157.500451:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00000001:3.0:1787764157.500454:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:2.0:1787764157.500455:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764157.500457:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764157.500458:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764157.500460:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764157.500461:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764157.500464:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764157.500465:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764157.500467:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764157.500468:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764157.500472:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:3.0:1787764157.500472:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832696704:192.168.203.110@tcp:250:kworker.0
00000008:00100000:2.0:1787764157.500476:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787764157.500477:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:2.0:1787764157.500480:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764157.500481:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1787764157.500482:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764157.500485:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764157.500487:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
02000000:00000001:3.0:1787764157.500489:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:2.0:1787764157.500490:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764157.500494:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764157.500495:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764157.500496:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000010:3.0:1787764157.500497:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612c00.
02000000:00000001:2.0:1787764157.500498:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764157.500500:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764157.500503:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764157.500505:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5b9800.
00000100:00000001:2.0:1787764157.500509:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764157.500513:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00100000:2.0:1787764157.500518:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832696832/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1787764157.500521:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2345d8.
00000100:00000001:2.0:1787764157.500530:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:3.0:1787764157.500531:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32aa0.
00000100:00000001:2.0:1787764157.500534:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764157.500540:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832696704, portal 25
00000100:00000040:2.0:1787764157.500542:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832696832/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764157.500544:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764157.500546:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:3.0:1787764157.500555:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832696704/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764182 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764157.500557:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764157.500563:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764157.500565:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:2.0:1787764157.500569:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000040:3.0:1787764157.500570:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787764157.500574:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:3.0:1787764157.500575:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00080000:2.0:1787764157.500576:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:3.0:1787764157.500578:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832696704, offset 0
00000100:00000001:2.0:1787764157.500579:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764157.500583:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21872
00000400:00000200:3.0:1787764157.500587:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764157.500590:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764157.500594:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764157.500596:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1787764157.500598:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:2.0:1787764157.500601:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764157.500604:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764157.500608:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:3.0:1787764157.500608:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:2.0:1787764157.500614:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764157.500615:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:3.0:1787764157.500616:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:2.0:1787764157.500617:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787764157.500620:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:3.0:1787764157.500622:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000020:00000040:2.0:1787764157.500623:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:2.0:1787764157.500625:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:3.0:1787764157.500627:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000020:00000001:2.0:1787764157.500629:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:3.0:1787764157.500632:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000040:2.0:1787764157.500635:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764157.500638:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787764157.500640:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58678:58678:256:4294967295] 192.168.203.110@tcp LPNI seq info [58678:58678:8:4294967295]
00000100:00000040:2.0:1787764157.500642:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764157.500646:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:3.0:1787764157.500647:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:2.0:1787764157.500649:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:2.0:1787764157.500652:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764157.500653:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764157.500656:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764157.500659:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1787764157.500660:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
02000000:00000001:2.0:1787764157.500663:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764157.500666:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764157.500669:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000800:00000200:3.0:1787764157.500669:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000008:00000001:2.0:1787764157.500674:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764157.500676:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1787764157.500676:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000100:00000010:2.0:1787764157.500679:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000020:00000040:2.0:1787764157.500683:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000800:00000200:3.0:1787764157.500684:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:2.0:1787764157.500687:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764157.500690:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787764157.500690:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787764157.500692:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764157.500695:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764157.500697:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764157.500700:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5ba000.
00000100:00000001:2.0:1787764157.500702:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764157.500706:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832696960/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764157.500707:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764157.500715:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764157.500717:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764157.500718:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316545779
00000100:00000001:2.0:1787764157.500721:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764157.500725:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832696960/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1787764157.500725:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40a20.
00000100:00000001:2.0:1787764157.500733:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764157.500734:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000040:2.0:1787764157.500738:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1787764157.500741:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1787764157.500743:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316545779
00000100:00080000:2.0:1787764157.500744:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764157.500748:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000010:3.0:1787764157.500748:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d406c0.
00000100:00000001:2.0:1787764157.500752:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764157.500753:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764157.500753:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787764157.500758:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764157.500758:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764157.500763:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764157.500765:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764157.500767:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1787764157.500769:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21872
00000400:00000200:2.0:1787764157.500773:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764157.500774:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000400:00000200:2.0:1787764157.500776:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787764157.500777:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00080000:2.0:1787764157.500779:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787764157.500782:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1787764157.500783:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764157.500786:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764157.500786:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764157.500790:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764157.500791:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764157.500794:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764157.500796:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764157.500798:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787764157.500800:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:3.0:1787764157.500800:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832696832/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787764157.500802:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787764157.500805:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:2.0:1787764157.500809:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:2.0:1787764157.500815:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764157.500819:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764157.500821:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:2.0:1787764157.500822:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764157.500825:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764157.500825:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764157.500830:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:2.0:1787764157.500831:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
02000000:00000001:3.0:1787764157.500834:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:2.0:1787764157.500835:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:3.0:1787764157.500839:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764157.500840:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764157.500843:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:3.0:1787764157.500844:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832696832:192.168.203.110@tcp:38:kworker.0
02000000:00000001:2.0:1787764157.500846:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764157.500849:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764157.500850:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764157.500852:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1787764157.500852:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764157.500855:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:2.0:1787764157.500856:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
02000000:00000001:3.0:1787764157.500857:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:2.0:1787764157.500860:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764157.500863:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764157.500863:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612800.
02000000:00000001:3.0:1787764157.500868:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764157.500870:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000020:00000040:2.0:1787764157.500874:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000400:00000200:3.0:1787764157.500874:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:2.0:1787764157.500878:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:3.0:1787764157.500879:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc234738.
02000000:00000001:2.0:1787764157.500880:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764157.500883:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764157.500885:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:3.0:1787764157.500885:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32110.
02000000:00000001:2.0:1787764157.500886:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764157.500890:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832696832, portal 10
02000000:00000010:2.0:1787764157.500892:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bbc00.
00000100:00000001:3.0:1787764157.500894:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764157.500896:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000001:2.0:1787764157.500897:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764157.500903:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832697088/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:3.0:1787764157.500905:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832696832/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764182 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764157.500922:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764157.500924:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1787764157.500926:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764157.500930:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764157.500933:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000040:2.0:1787764157.500934:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832697088/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764157.500937:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832696832, offset 0
00000400:00000200:3.0:1787764157.500946:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787764157.500947:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764157.500951:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:2.0:1787764157.500952:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21882)
00000400:00000200:3.0:1787764157.500956:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764157.500959:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764157.500961:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764157.500962:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764157.500964:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764157.500966:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58679:58679:255:4294967295] 192.168.203.110@tcp LPNI seq info [58679:58679:7:4294967295]
00000400:00000200:3.0:1787764157.500969:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764157.500973:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764157.500977:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764157.500980:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186800.
00000800:00000200:3.0:1787764157.500984:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764157.500988:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787764157.500990:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764157.500993:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764157.500996:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764157.500998:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316544543
00000100:00000001:3.0:1787764157.501000:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316545779
00000100:00000001:3.0:1787764157.501002:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316544543
00000100:00000001:3.0:1787764157.501004:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316545779
00000100:00000001:3.0:1787764157.501006:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764157.501007:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764157.501009:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764157.501013:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832696960/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764157.501019:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764157.501020:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764157.501022:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764157.501023:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764157.501024:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764157.501027:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832696960:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764157.501030:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764157.501032:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764157.501033:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764157.501035:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764157.501038:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613000.
02000000:00000001:3.0:1787764157.501040:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764157.501042:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764157.501044:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc234e18.
00000400:00000010:3.0:1787764157.501047:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32bb0.
00000100:00000200:3.0:1787764157.501050:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832696960, portal 10
00000100:00000001:3.0:1787764157.501052:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764157.501053:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:3.0:1787764157.501059:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832696960/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764182 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764157.501065:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764157.501067:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764157.501070:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764157.501071:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832696960, offset 0
00000400:00000200:3.0:1787764157.501074:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764157.501077:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764157.501080:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764157.501081:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764157.501083:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764157.501084:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764157.501086:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764157.501088:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58680:58680:254:4294967295] 192.168.203.110@tcp LPNI seq info [58680:58680:6:4294967295]
00000400:00000200:3.0:1787764157.501090:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764157.501094:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764157.501096:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764157.501098:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764157.501101:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764157.501103:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787764157.501105:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764157.501106:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764157.501108:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764157.501110:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316544531
00000100:00000001:3.0:1787764157.501111:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316545779
00000100:00000001:3.0:1787764157.501112:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316544531
00000100:00000001:3.0:1787764157.501114:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316545779
00000100:00000001:3.0:1787764157.501115:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764157.501116:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764157.501118:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764157.501120:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764157.501123:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764157.501124:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1787764157.501126:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764157.501127:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764157.501128:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764157.501132:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832697088/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764157.501175:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764157.501176:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764157.501178:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764157.501179:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764157.501181:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764157.501184:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832697088:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764157.501187:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764157.501188:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764157.501190:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764157.501192:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764157.501195:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612400.
02000000:00000001:3.0:1787764157.501196:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764157.501199:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764157.501202:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2340b0.
00000400:00000010:3.0:1787764157.501205:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32d48.
00000100:00000200:3.0:1787764157.501207:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832697088, portal 4
00000100:00000001:3.0:1787764157.501209:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764157.501210:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:3.0:1787764157.501215:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832697088/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764182 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764157.501221:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764157.501223:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764157.501226:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764157.501228:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832697088, offset 0
00000400:00000200:3.0:1787764157.501232:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764157.501236:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764157.501239:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764157.501241:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764157.501243:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764157.501245:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764157.501247:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764157.501250:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58681:58681:253:4294967295] 192.168.203.110@tcp LPNI seq info [58681:58681:5:4294967295]
00000400:00000200:3.0:1787764157.501253:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764157.501257:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764157.501261:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764157.501263:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186500.
00000800:00000200:3.0:1787764157.501267:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764157.501270:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787764157.501272:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764157.501274:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764157.501277:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764157.501279:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316544549
00000100:00000001:3.0:1787764157.501281:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316545779
00000100:00000001:3.0:1787764157.501283:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316544549
00000100:00000001:3.0:1787764157.501286:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316545779
00000100:00000001:3.0:1787764157.501288:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764157.501290:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764157.501292:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764157.501293:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764157.501296:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764157.501297:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1787764157.501299:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764157.501300:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764157.501301:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764157.501303:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764158.140239:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764158.140240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764158.140246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764158.140248:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764158.140253:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764158.140255:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764158.140258:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764158.140259:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764158.140264:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764158.140265:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764158.140268:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764158.140270:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764158.140272:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764158.140275:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764158.140275:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764158.140279:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764158.140280:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764158.140282:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764158.140286:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764158.140288:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764158.140290:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764158.140338:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764158.140342:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764158.140346:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764158.524240:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764158.524247:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764158.524250:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764158.524254:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764158.524258:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764158.524262:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764158.524265:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764158.524267:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764158.524270:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764158.780234:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316545779, at 4316545792
00000100:00000010:3.0:1787764158.780246:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d406c0.
00000100:00000040:2.0:1787764158.780261:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316545779, at 4316545792
00000100:00000010:2.0:1787764158.780268:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40a20.
00010000:00000001:3.0:1787764159.164223:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764159.164228:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764159.164233:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764159.164233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764159.164235:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764159.164240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764159.164241:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764159.164242:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764159.164245:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764159.164250:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764159.164254:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764159.164257:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764159.164261:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764159.164264:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1787764159.164267:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000100:00000001:2.0:1787764159.164267:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764159.164273:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764159.164277:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764159.548213:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764159.548221:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764159.548225:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764159.548230:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764159.548235:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764159.548239:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764159.548242:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764159.548245:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764159.548248:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764160.188228:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764160.188231:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764160.188233:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764160.188234:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764160.188236:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764160.188239:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764160.188241:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764160.188242:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764160.188246:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764160.188248:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764160.188250:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764160.188251:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764160.188253:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764160.188256:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764160.188258:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764160.188274:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764160.188278:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764160.188281:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764160.572226:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764160.572233:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764160.572235:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764160.572240:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764160.572244:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764160.572246:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764160.572249:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764160.572251:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764160.572254:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:3.0:1787764160.572407:0:257564:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.110:988
00000400:00000100:3.0:1787764160.572439:0:257564:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.110@tcp at host 192.168.203.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:3.0:1787764160.572458:0:257564:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764160.572462:0:257564:0:(router.c:1791:lnet_notify()) 192.168.203.10@tcp notifying 192.168.203.110@tcp: down
00000400:00000200:3.0:1787764160.572465:0:257564:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764160.572469:0:257564:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:3.0:1787764160.572474:0:257564:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:3.0:1787764160.572477:0:257564:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:3.0:1787764160.572483:0:257564:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:3.0:1787764160.572488:0:257564:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:3.0:1787764160.572494:0:257564:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:3.0:1787764160.572496:0:257564:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:3.0:1787764160.572502:0:257564:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764160.572510:0:257564:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f86583c0 x1874599832696704/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764182 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764160.572535:0:257564:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764160.572539:0:257564:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832696704/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764182 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764160.572546:0:257564:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764160.572548:0:257564:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:3.0:1787764160.572557:0:257564:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:3.0:1787764160.572559:0:257564:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186800.
00000400:00000200:3.0:1787764160.572561:0:257564:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:3.0:1787764160.572564:0:257564:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:3.0:1787764160.572567:0:257564:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764160.572569:0:257564:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764160.572571:0:257564:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764160.572576:0:257564:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f8658000 x1874599832696832/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764182 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764160.572584:0:257564:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764160.572587:0:257564:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832696832/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764182 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764160.572590:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764160.572596:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764160.572596:0:257564:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764160.572601:0:257564:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764160.572602:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000800:00000100:3.0:1787764160.572609:0:257564:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000100:00100000:2.0:1787764160.572614:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764157/real 1787764160]  req@ffff9751f8658000 x1874599832696832/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:3.0:1787764160.572614:0:257564:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764160.572618:0:257564:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:3.0:1787764160.572626:0:257564:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:3.0:1787764160.572630:0:257564:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764160.572632:0:257564:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764160.572637:0:257564:0:(events.c:34:request_out_callback()) Process entered
00000400:00000001:2.0:1787764160.572643:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000200:3.0:1787764160.572644:0:257564:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f8659a40 x1874599832696960/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764182 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764160.572646:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764160.572652:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc234738.
00000400:00000200:2.0:1787764160.572656:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32110
00000400:00000010:2.0:1787764160.572660:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32110.
00000100:00000001:2.0:1787764160.572666:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:3.0:1787764160.572668:0:257564:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000200:2.0:1787764160.572675:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832696832/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787764160.572676:0:257564:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832696960/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764182 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764160.572703:0:257564:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764160.572707:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832696832/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764160.572707:0:257564:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:3.0:1787764160.572718:0:257564:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:3.0:1787764160.572724:0:257564:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186500.
00000400:00000200:3.0:1787764160.572729:0:257564:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00000001:2.0:1787764160.572733:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000400:00000200:3.0:1787764160.572736:0:257564:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000100:00000001:2.0:1787764160.572737:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764160.572742:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764160.572744:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:3.0:1787764160.572747:0:257564:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000100:00100000:2.0:1787764160.572750:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832696832/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1787764160.572751:0:257564:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764160.572757:0:257564:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:2.0:1787764160.572763:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000200:3.0:1787764160.572765:0:257564:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f865ad00 x1874599832697088/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764182 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787764160.572772:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832696832/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764160.572777:0:257564:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764160.572783:0:257564:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832697088/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764182 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764160.572803:0:257564:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764160.572804:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764160.572806:0:257564:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764160.572814:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764160.572817:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764160.572818:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764160.572821:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764160.572834:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764160.572838:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764160.572843:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764160.572846:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764160.572850:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832696832/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764160.572861:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832696832:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764160.572866:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764160.572870:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764157/real 1787764160]  req@ffff9751f8659a40 x1874599832696960/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764160.572877:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764160.572878:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764160.572880:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc234e18.
00000400:00000200:2.0:1787764160.572883:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32bb0
00000400:00000010:2.0:1787764160.572884:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32bb0.
00000100:00000001:2.0:1787764160.572886:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764160.572889:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832696960/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764160.572895:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832696960/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764160.572901:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764160.572902:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764160.572904:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764160.572905:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764160.572908:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832696960/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764160.572913:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764160.572916:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832696960/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764160.572921:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764160.572923:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764160.572925:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764160.572926:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764160.572927:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764160.572928:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764160.572931:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764160.572934:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764160.572936:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764160.572938:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832696960/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764160.572945:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832696960:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764160.572949:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764160.572953:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764157/real 1787764160]  req@ffff9751f86583c0 x1874599832696704/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764160.572960:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764160.572961:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764160.572963:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2345d8.
00000400:00000200:2.0:1787764160.572965:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32aa0
00000400:00000010:2.0:1787764160.572967:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32aa0.
00000100:00000001:2.0:1787764160.572969:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764160.572972:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832696704/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764160.572993:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832696704/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764160.573000:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764160.573001:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764160.573003:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764160.573004:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764160.573007:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832696704/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764160.573013:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764160.573016:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832696704/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764160.573021:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764160.573023:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764160.573024:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764160.573025:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764160.573027:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764160.573028:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764160.573030:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764160.573034:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764160.573036:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764160.573039:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832696704/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764160.573046:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832696704:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764160.573049:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764160.573051:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764160.573053:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832696832/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764160.573060:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764160.573062:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764160.573067:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612800.
02000000:00000001:2.0:1787764160.573070:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764160.573081:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764160.573084:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764160.573086:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764160.573088:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764160.573092:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764160.573094:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764160.573095:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:2.0:1787764160.573098:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764160.573099:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764160.573100:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764160.573103:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832696960/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764160.573108:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764160.573110:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764160.573111:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613000.
02000000:00000001:2.0:1787764160.573113:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764160.573115:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764160.573116:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764160.573127:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764160.573129:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764160.573131:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764160.573132:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764160.573151:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:2.0:1787764160.573154:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764160.573155:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764160.573157:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764160.573160:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832696704/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764160.573166:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764160.573167:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764160.573168:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612c00.
02000000:00000001:2.0:1787764160.573171:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764160.573172:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764160.573173:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764160.573175:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764160.573176:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764160.573178:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764160.573179:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764160.573180:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:2.0:1787764160.573182:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764160.573183:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764160.573191:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764160.573193:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764160.573194:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764160.573196:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764160.573200:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764157/real 1787764160]  req@ffff9751f865ad00 x1874599832697088/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764160.573208:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764160.573209:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764160.573211:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2340b0.
00000400:00000200:2.0:1787764160.573213:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32d48
00000400:00000010:2.0:1787764160.573215:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32d48.
00000100:00000001:2.0:1787764160.573217:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764160.573219:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832697088/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764160.573232:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832697088/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764160.573249:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764160.573251:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764160.573253:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764160.573254:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764160.573257:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832697088/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764160.573262:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764160.573266:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832697088/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764160.573272:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764160.573274:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764160.573275:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764160.573277:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764160.573278:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764160.573280:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764160.573282:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764160.573285:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764160.573287:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764160.573291:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832697088/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764160.573299:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832697088:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764160.573304:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764160.573305:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764160.573308:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832697088/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764182 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764160.573319:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764160.573320:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764160.573322:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612400.
02000000:00000001:2.0:1787764160.573325:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764160.573327:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764160.573328:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764160.573329:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764160.573330:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764160.573332:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764160.573333:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764160.573334:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:2.0:1787764160.573336:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764160.573337:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764160.573339:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764160.573351:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764160.573353:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764160.573355:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764160.573356:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764161.212218:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764161.212226:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764161.212229:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764161.212234:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764161.212239:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764161.212240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764161.212243:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764161.212249:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764161.212249:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764161.212253:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764161.212259:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764161.212259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764161.212263:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764161.212269:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764161.212272:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764161.212312:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764161.212315:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764161.212318:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764161.596230:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764161.596236:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764161.596238:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764161.596242:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764161.596245:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764161.596249:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764161.596251:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764161.596252:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764161.596254:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764162.236235:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764162.236235:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764162.236241:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764162.236243:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764162.236246:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764162.236248:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764162.236248:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764162.236255:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764162.236255:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764162.236257:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764162.236261:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764162.236266:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764162.236270:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764162.236273:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764162.236276:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764162.236281:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764162.236286:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764162.236289:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764162.620233:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764162.620241:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764162.620244:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764162.620250:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764162.620263:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764162.620267:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.620269:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764162.620271:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.620274:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764162.620288:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764162.620295:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787764162.620297:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764162.620300:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764162.620304:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764162.620311:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21877
00000400:00000200:1.0:1787764162.620319:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764162.620325:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764162.620329:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764162.620333:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764162.620336:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764162.620339:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764162.620341:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764162.620346:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764162.620348:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787764162.620351:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764162.620355:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787764162.620357:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787764162.620360:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787764162.620363:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764162.620366:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764162.620369:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764162.620371:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764162.620374:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787764162.620378:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764162.620381:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764162.620384:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764162.620387:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764162.620391:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764162.620393:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764162.620396:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764162.620402:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7931680.
00000020:00000040:1.0:1787764162.620405:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787764162.620412:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764162.620414:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764162.620417:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764162.620420:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764162.620424:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764162.620430:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8000.
00000100:00000001:1.0:1787764162.620436:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764162.620445:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7931680 x1874599832697216/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764162.620454:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764162.620458:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764162.620463:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d4255d6] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7931680 x1874599832697216/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764162.620486:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764162.620490:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764162.620493:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787764162.620495:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764162.620496:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764162.620499:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764162.620501:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21877
00000400:00000200:1.0:1787764162.620505:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764162.620508:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764162.620511:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764162.620514:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764162.620516:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764162.620518:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764162.620520:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764162.620522:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764162.620524:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00000001:3.0:1787764162.620524:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:1.0:1787764162.620528:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787764162.620530:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000040:1.0:1787764162.620531:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000001:3.0:1787764162.620534:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1787764162.620535:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787764162.620539:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:3.0:1787764162.620543:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7931680 x1874599832697216/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787764162.620544:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764162.620549:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764162.620552:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764162.620557:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764162.620562:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:1.0:1787764162.620563:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00000001:3.0:1787764162.620566:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764162.620568:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:3.0:1787764162.620571:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:1.0:1787764162.620572:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764162.620576:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764162.620576:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764162.620581:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764162.620584:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:3.0:1787764162.620587:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832697216:192.168.203.110@tcp:250:kworker.0
02000000:00000001:1.0:1787764162.620588:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764162.620594:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764162.620596:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:1.0:1787764162.620599:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1787764162.620601:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00000001:1.0:1787764162.620605:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764162.620606:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764162.620609:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764162.620611:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000010:1.0:1787764162.620617:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930b40.
00000020:00000040:1.0:1787764162.620620:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764162.620623:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764162.620624:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764162.620626:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000010:3.0:1787764162.620627:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613c00.
02000000:00000001:1.0:1787764162.620628:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764162.620631:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764162.620633:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764162.620638:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb400.
00000400:00000200:3.0:1787764162.620640:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764162.620645:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764162.620647:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000100:00100000:1.0:1787764162.620654:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930b40 x1874599832697344/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1787764162.620656:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764162.620665:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832697216, portal 25
00000100:00000001:1.0:1787764162.620666:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764162.620670:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764162.620671:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764162.620672:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497727104 : -115097211824512 : ffff9751d7931680)
00000100:00000040:1.0:1787764162.620677:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022d7f91a] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930b40 x1874599832697344/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787764162.620682:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7931680 x1874599832697216/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764187 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764162.620695:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764162.620698:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:1.0:1787764162.620700:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1787764162.620704:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00080000:1.0:1787764162.620707:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000010:3.0:1787764162.620709:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000001:1.0:1787764162.620710:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764162.620712:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764162.620715:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764162.620717:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21877
00000100:00000200:3.0:1787764162.620722:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832697216, offset 0
00000400:00000200:1.0:1787764162.620723:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764162.620728:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764162.620731:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1787764162.620733:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000040:1.0:1787764162.620738:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764162.620741:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787764162.620744:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:1.0:1787764162.620746:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764162.620749:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764162.620752:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764162.620755:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000400:00000200:3.0:1787764162.620755:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000020:00000001:1.0:1787764162.620759:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764162.620763:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000400:00000200:3.0:1787764162.620764:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:1.0:1787764162.620767:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787764162.620770:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:3.0:1787764162.620771:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000040:1.0:1787764162.620776:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:3.0:1787764162.620779:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1787764162.620780:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787764162.620784:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000040:1.0:1787764162.620785:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764162.620789:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:3.0:1787764162.620792:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58682:58682:256:4294967295] 192.168.203.110@tcp LPNI seq info [58682:58682:8:4294967295]
00000020:00000040:1.0:1787764162.620793:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787764162.620800:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:3.0:1787764162.620803:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764162.620805:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764162.620808:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764162.620810:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764162.620815:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764162.620820:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1787764162.620820:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000008:00100000:1.0:1787764162.620824:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787764162.620827:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787764162.620830:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787764162.620831:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764162.620837:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930000.
00000800:00000010:3.0:1787764162.620839:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186500.
00000020:00000040:1.0:1787764162.620841:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764162.620845:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:3.0:1787764162.620848:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:1.0:1787764162.620849:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764162.620852:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764162.620855:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:3.0:1787764162.620856:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
02000000:00000001:1.0:1787764162.620857:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764162.620865:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9000.
00000800:00000200:3.1:1787764162.620865:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764162.620871:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:1.0:1787764162.620872:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1787764162.620880:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00100000:1.0:1787764162.620882:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930000 x1874599832697472/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000010:3.0:1787764162.620884:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186500.
00000400:00000200:3.0:1787764162.620891:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:1.0:1787764162.620896:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764162.620900:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764162.620901:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:1.0:1787764162.620912:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6d90274] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930000 x1874599832697472/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764162.620921:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764162.620922:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000100:00000040:1.0:1787764162.620925:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:3.0:1787764162.620926:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000040:1.0:1787764162.620933:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1787764162.620934:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:1.0:1787764162.620937:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764162.620942:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000200:3.0:1787764162.620942:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7931680 x1874599832697216/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764187 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764162.620943:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764162.620946:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764162.620950:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764162.620956:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21877
00000100:00000001:3.0:1787764162.620961:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000400:00000200:1.0:1787764162.620962:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764162.620967:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000040:3.0:1787764162.620968:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7931680 x1874599832697216/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764187 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787764162.620970:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764162.620976:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764162.620981:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764162.620985:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764162.620988:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764162.620990:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764162.620992:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787764162.620993:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764162.620994:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787764162.620996:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787764162.620999:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:1.0:1787764162.621001:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764162.621003:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764162.621005:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764162.621007:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764162.621009:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:1.0:1787764162.621011:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764162.621013:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764162.621014:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764162.621017:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764162.621018:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764162.621020:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764162.621022:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1787764162.621024:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764162.621026:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764162.621029:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933480.
00000020:00000040:1.0:1787764162.621031:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787764162.621034:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764162.621035:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764162.621036:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764162.621038:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764162.621039:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764162.621042:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca000.
00000100:00000001:1.0:1787764162.621045:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764162.621049:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933480 x1874599832697600/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764162.621055:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764162.621057:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764162.621060:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3c2c33c] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933480 x1874599832697600/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764162.621067:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764162.621071:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21887)
00000100:00000001:3.0:1787764162.621882:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764162.621887:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764162.621894:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764162.622060:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764162.622063:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316550900
00000100:00000010:3.0:1787764162.622069:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40a20.
00000100:00000001:3.0:1787764162.622074:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787764162.622080:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316550900
00000100:00000010:3.0:1787764162.622083:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40420.
00000100:00000001:3.0:1787764162.622086:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787764162.622089:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764162.622091:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764162.622094:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764162.622101:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764162/real 1787764162]  req@ffff9751d7931680 x1874599832697216/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764162.622124:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764162.622126:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764162.622129:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764162.622185:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764162.622187:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764162.622190:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764162.622194:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7931680 x1874599832697216/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764162.622203:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7931680 x1874599832697216/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.622210:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764162.622212:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.622215:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764162.622216:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764162.622221:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7931680 x1874599832697216/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.622228:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764162.622233:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7931680 x1874599832697216/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.622242:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764162.622245:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.622247:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764162.622247:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.622250:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764162.622252:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764162.622255:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764162.622259:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764162.622261:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764162.622265:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7931680 x1874599832697216/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764162.622274:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832697216:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764162.622279:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.622290:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764162.622294:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7931680 x1874599832697216/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.622302:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764162.622304:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764162.622307:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613c00.
02000000:00000001:3.0:1787764162.622310:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764162.622312:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764162.622314:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764162.622316:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764162.622318:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8000.
02000000:00000001:3.0:1787764162.622322:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764162.622324:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764162.622325:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7931680.
00000100:00000001:3.0:1787764162.622328:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764162.622330:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.622332:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.622338:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764162.622340:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764162.622342:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764162.622344:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764162.622346:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764162.622350:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930b40 x1874599832697344/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.622357:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764162.622358:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764162.622361:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764162.622363:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764162.622366:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764162.622369:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832697344:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764162.622372:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764162.622375:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764162.622377:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764162.622379:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764162.622383:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613c00.
02000000:00000001:3.0:1787764162.622384:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764162.622388:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764162.622391:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764162.622395:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764162.622398:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832697344, portal 10
00000100:00000001:3.0:1787764162.622401:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764162.622402:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497724224 : -115097211827392 : ffff9751d7930b40)
00000100:00000040:3.0:1787764162.622408:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930b40 x1874599832697344/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764187 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.622415:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764162.622417:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764162.622419:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764162.622421:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832697344, offset 0
00000400:00000200:3.0:1787764162.622426:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764162.622431:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764162.622435:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764162.622439:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764162.622443:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764162.622446:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764162.622448:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764162.622452:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58683:58683:256:4294967295] 192.168.203.110@tcp LPNI seq info [58683:58683:8:4294967295]
00000400:00000200:3.0:1787764162.622456:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764162.622463:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764162.622468:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764162.622472:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186500.
00000800:00000200:3.0:1787764162.622477:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764162.622480:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787764162.622485:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764162.622488:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1787764162.622492:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764162.622494:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186500.
00000400:00000200:3.0:1787764162.622497:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764162.622501:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764162.622504:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764162.622506:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764162.622509:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764162.622512:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930b40 x1874599832697344/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764187 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.622525:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764162.622528:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930b40 x1874599832697344/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764187 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.622534:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764162.622536:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764162.622538:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764162.622542:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764162.622545:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764162.622547:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316549665
00000100:00000001:3.0:1787764162.622550:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316550900
00000100:00000001:3.0:1787764162.622552:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316549665
00000100:00000001:3.0:1787764162.622555:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316550900
00000100:00000001:3.0:1787764162.622557:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764162.622558:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764162.622560:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764162.622563:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764162/real 1787764162]  req@ffff9751d7930b40 x1874599832697344/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764162.622570:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764162.622572:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764162.622573:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764162.622575:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764162.622577:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764162.622578:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764162.622581:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930b40 x1874599832697344/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764162.622589:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930b40 x1874599832697344/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.622595:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764162.622597:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.622599:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764162.622600:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764162.622604:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930b40 x1874599832697344/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.622609:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764162.622613:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930b40 x1874599832697344/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.622620:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764162.622623:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.622624:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764162.622625:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.622627:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764162.622639:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764162.622642:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764162.622644:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764162.622646:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764162.622650:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930b40 x1874599832697344/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764162.622657:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832697344:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764162.622661:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764162.622664:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930000 x1874599832697472/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.622671:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764162.622672:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764162.622674:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764162.622676:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764162.622679:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764162.622683:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832697472:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764162.622686:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764162.622688:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764162.622689:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764162.622691:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764162.622694:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611800.
02000000:00000001:3.0:1787764162.622696:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764162.622699:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764162.622702:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764162.622705:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764162.622708:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832697472, portal 10
00000100:00000001:3.0:1787764162.622710:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764162.622711:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497721344 : -115097211830272 : ffff9751d7930000)
00000100:00000040:3.0:1787764162.622717:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930000 x1874599832697472/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764187 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.622770:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764162.622772:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764162.622774:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764162.622777:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832697472, offset 0
00000400:00000200:3.0:1787764162.622781:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764162.622785:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764162.622788:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764162.622791:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764162.622793:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764162.622795:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764162.622798:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764162.622801:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58684:58684:256:4294967295] 192.168.203.110@tcp LPNI seq info [58684:58684:8:4294967295]
00000400:00000200:3.0:1787764162.622805:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764162.622811:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764162.622815:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764162.622818:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186500.
00000800:00000200:3.0:1787764162.622822:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764162.622825:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787764162.622829:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764162.622832:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1787764162.622835:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764162.622837:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186500.
00000400:00000200:3.0:1787764162.622840:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764162.622843:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764162.622846:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764162.622847:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764162.622849:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764162.622853:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930000 x1874599832697472/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764187 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.622873:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764162.622876:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930000 x1874599832697472/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764187 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.622882:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764162.622884:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764162.622887:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764162.622889:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764162.622892:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764162.622894:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316549653
00000100:00000001:3.0:1787764162.622896:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316550900
00000100:00000001:3.0:1787764162.622898:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316549653
00000100:00000001:3.0:1787764162.622900:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316550900
00000100:00000001:3.0:1787764162.622908:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764162.622910:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764162.622912:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764162.622915:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764162/real 1787764162]  req@ffff9751d7930000 x1874599832697472/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764162.622922:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764162.622924:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764162.622926:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764162.622928:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764162.622930:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764162.622932:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764162.622935:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930000 x1874599832697472/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764162.622943:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930000 x1874599832697472/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.622948:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764162.622949:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.622951:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764162.622952:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764162.622958:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930000 x1874599832697472/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.622964:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764162.622968:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930000 x1874599832697472/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.622974:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764162.622977:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.622978:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764162.622979:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.622980:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764162.622982:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764162.622984:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764162.622988:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764162.622990:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764162.622993:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930000 x1874599832697472/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764162.623007:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832697472:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764162.623011:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764162.623014:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933480 x1874599832697600/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.623021:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764162.623022:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764162.623023:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764162.623025:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764162.623027:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764162.623030:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832697600:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764162.623033:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764162.623035:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764162.623036:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764162.623037:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764162.623040:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8613800.
02000000:00000001:3.0:1787764162.623042:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764162.623045:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764162.623047:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764162.623050:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764162.623053:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832697600, portal 4
00000100:00000001:3.0:1787764162.623054:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764162.623056:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497734784 : -115097211816832 : ffff9751d7933480)
00000100:00000040:3.0:1787764162.623061:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933480 x1874599832697600/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764187 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.623068:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764162.623070:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764162.623072:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764162.623074:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832697600, offset 0
00000400:00000200:3.0:1787764162.623077:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764162.623081:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764162.623084:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764162.623087:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764162.623089:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764162.623091:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764162.623093:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764162.623096:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58685:58685:256:4294967295] 192.168.203.110@tcp LPNI seq info [58685:58685:8:4294967295]
00000400:00000200:3.0:1787764162.623100:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764162.623118:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764162.623122:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764162.623126:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186500.
00000800:00000200:3.0:1787764162.623129:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764162.623186:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787764162.623191:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764162.623194:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1787764162.623197:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764162.623199:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186500.
00000400:00000200:3.0:1787764162.623202:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764162.623205:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764162.623207:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764162.623209:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764162.623211:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764162.623215:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933480 x1874599832697600/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764187 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.623222:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764162.623225:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933480 x1874599832697600/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764187 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.623232:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764162.623233:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764162.623235:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764162.623238:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764162.623241:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764162.623243:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316549671
00000100:00000001:3.0:1787764162.623246:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316550900
00000100:00000001:3.0:1787764162.623248:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316549671
00000100:00000001:3.0:1787764162.623250:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316550900
00000100:00000001:3.0:1787764162.623252:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764162.623254:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764162.623255:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764162.623259:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764162/real 1787764162]  req@ffff9751d7933480 x1874599832697600/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764162.623267:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764162.623269:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764162.623270:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764162.623272:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764162.623274:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764162.623276:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764162.623278:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933480 x1874599832697600/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764162.623286:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933480 x1874599832697600/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.623292:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764162.623294:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.623295:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764162.623296:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764162.623300:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933480 x1874599832697600/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.623306:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764162.623310:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933480 x1874599832697600/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.623316:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764162.623319:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.623320:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764162.623321:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.623323:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764162.623324:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764162.623327:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764162.623330:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764162.623332:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764162.623335:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933480 x1874599832697600/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764162.623342:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832697600:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764162.623346:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.623348:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764162.623351:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930b40 x1874599832697344/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.623357:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764162.623359:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764162.623360:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613c00.
02000000:00000001:3.0:1787764162.623363:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764162.623364:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764162.623365:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764162.623367:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764162.623369:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb400.
02000000:00000001:3.0:1787764162.623371:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764162.623373:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764162.623374:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930b40.
00000100:00000001:3.0:1787764162.623376:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764162.623377:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.623379:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764162.623381:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930000 x1874599832697472/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.623393:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764162.623394:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764162.623396:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611800.
02000000:00000001:3.0:1787764162.623397:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764162.623399:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764162.623400:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764162.623402:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764162.623403:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9000.
02000000:00000001:3.0:1787764162.623405:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764162.623406:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764162.623407:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930000.
00000100:00000001:3.0:1787764162.623408:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764162.623409:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.623411:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764162.623413:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933480 x1874599832697600/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764187 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764162.623420:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764162.623421:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764162.623422:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8613800.
02000000:00000001:3.0:1787764162.623424:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764162.623425:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764162.623427:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764162.623428:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764162.623429:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca000.
02000000:00000001:3.0:1787764162.623431:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764162.623432:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764162.623433:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933480.
00000100:00000001:3.0:1787764162.623436:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764162.623437:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.623439:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764162.623442:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764162.623443:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764162.623445:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764162.623447:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764163.260226:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764163.260232:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764163.260232:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764163.260235:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764163.260236:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764163.260240:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764163.260243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764163.260244:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764163.260247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764163.260248:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764163.260252:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764163.260254:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764163.260255:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764163.260257:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764163.260261:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764163.260264:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764163.260268:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764163.260271:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764163.260274:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764163.260276:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764163.260279:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764163.260317:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764163.260321:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764163.260323:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764163.644236:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764163.644244:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764163.644246:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764163.644251:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764163.644255:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764163.644258:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764163.644261:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764163.644263:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764163.644266:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764163.900224:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316550900, at 4316550912
00000100:00000010:3.0:1787764163.900238:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40420.
00000100:00000040:2.0:1787764163.900257:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316550900, at 4316550912
00000100:00000010:2.0:1787764163.900266:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40a20.
00000100:00000001:2.0:1787764164.284231:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764164.284236:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764164.284238:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764164.284241:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764164.284242:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764164.284248:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764164.284250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764164.284251:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764164.284253:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764164.284255:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764164.284258:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764164.284259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764164.284260:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764164.284262:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764164.284264:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764164.284303:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764164.284308:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764164.284311:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764164.668222:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764164.668230:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764164.668233:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764164.668239:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764164.668242:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764164.668246:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764164.668248:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764164.668250:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764164.668253:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764165.308200:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764165.308207:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764165.308210:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764165.308212:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764165.308215:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764165.308216:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764165.308219:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764165.308220:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764165.308224:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764165.308224:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764165.308229:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764165.308230:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764165.308233:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764165.308233:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764165.308236:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764165.308238:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764165.308242:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764165.308246:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764165.308248:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764165.308251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764165.308253:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764165.308291:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764165.308294:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764165.308296:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764165.692252:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764165.692260:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764165.692262:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764165.692268:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764165.692272:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764165.692276:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764165.692279:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764165.692281:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764165.692284:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764166.332239:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764166.332245:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764166.332251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764166.332252:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764166.332256:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764166.332258:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764166.332270:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000020:00000001:0.0:1787764166.332283:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000100:00000001:2.0:1787764166.332283:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000020:00000001:0.0:1787764166.332290:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764166.332292:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764166.332295:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00100000:2.0:1787764166.332301:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764166.332309:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764166.332315:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764166.332318:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764166.332321:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764166.332326:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764166.716222:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764166.716232:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764166.716234:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764166.716239:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764166.716242:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764166.716245:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764166.716248:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764166.716250:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764166.716252:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764167.356220:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764167.356222:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764167.356225:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764167.356231:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764167.356232:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764167.356236:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764167.356238:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764167.356239:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764167.356242:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764167.356246:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00100000:2.0:1787764167.356247:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764167.356248:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764167.356250:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764167.356254:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764167.356257:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764167.356258:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764167.356261:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764167.356265:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.356270:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764167.356274:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.356277:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764167.356297:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764167.356302:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764167.356305:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787764167.740245:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764167.740256:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764167.740258:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764167.740263:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764167.740265:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764167.740268:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1787764167.740271:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764167.740273:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764167.740277:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:3.0:1787764167.740278:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21882
00000100:00000001:0.0:1787764167.740283:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764167.740286:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:3.0:1787764167.740291:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764167.740292:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764167.740296:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764167.740300:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764167.740303:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764167.740309:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764167.740312:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764167.740316:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764167.740320:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764167.740323:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764167.740327:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764167.740329:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787764167.740332:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764167.740335:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787764167.740337:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:3.0:1787764167.740339:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:3.0:1787764167.740341:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764167.740343:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764167.740344:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764167.740347:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764167.740349:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:3.0:1787764167.740352:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764167.740353:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764167.740356:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764167.740358:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764167.740360:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764167.740362:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764167.740364:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764167.740371:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bf480.
00000020:00000040:3.0:1787764167.740374:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787764167.740377:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764167.740378:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764167.740381:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764167.740382:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764167.740386:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764167.740392:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613800.
00000100:00000001:3.0:1787764167.740397:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764167.740404:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bf480 x1874599832697728/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764167.740412:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764167.740415:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764167.740419:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c2965388] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bf480 x1874599832697728/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764167.740436:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764167.740439:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764167.740441:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787764167.740443:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764167.740444:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764167.740446:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764167.740448:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21882
00000400:00000200:3.0:1787764167.740451:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764167.740454:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764167.740455:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764167.740458:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764167.740460:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764167.740461:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764167.740463:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764167.740465:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764167.740466:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:3.0:1787764167.740467:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764167.740468:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:3.0:1787764167.740470:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:3.0:1787764167.740471:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:3.0:1787764167.740473:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764167.740474:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764167.740475:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764167.740480:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764167.740482:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764167.740483:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000040:3.0:1787764167.740484:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00000001:2.0:1787764167.740488:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00080000:3.0:1787764167.740488:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764167.740494:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764167.740495:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764167.740500:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1787764167.740502:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bf480 x1874599832697728/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787764167.740503:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764167.740508:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787764167.740513:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:3.0:1787764167.740520:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764167.740523:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764167.740524:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764167.740526:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764167.740532:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87be1c0.
02000000:00000001:2.0:1787764167.740533:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:3.0:1787764167.740538:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764167.740540:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764167.740544:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764167.740546:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764167.740548:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764167.740553:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00100000:2.0:1787764167.740555:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832697728:192.168.203.110@tcp:250:kworker.0
02000000:00000001:3.0:1787764167.740555:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764167.740559:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764167.740566:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000010:3.0:1787764167.740566:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611800.
00000100:00000001:3.0:1787764167.740573:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764167.740574:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764167.740579:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764167.740579:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87be1c0 x1874599832697856/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1787764167.740586:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1787764167.740590:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764167.740592:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764167.740596:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
00000100:00000040:3.0:1787764167.740596:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003e2011c6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87be1c0 x1874599832697856/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764167.740603:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764167.740606:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764167.740610:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787764167.740616:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:3.0:1787764167.740617:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1787764167.740622:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:2.0:1787764167.740623:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000100:00080000:3.0:1787764167.740625:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764167.740630:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764167.740634:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21882
00000400:00000010:2.0:1787764167.740635:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000400:00000200:3.0:1787764167.740641:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000200:2.0:1787764167.740646:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832697728, portal 25
00000400:00000200:3.0:1787764167.740647:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787764167.740652:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00080000:3.0:1787764167.740653:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787764167.740657:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049859200 : -115096659692416 : ffff9751f87bf480)
00000100:00000040:3.0:1787764167.740660:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764167.740664:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764167.740669:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000040:2.0:1787764167.740671:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bf480 x1874599832697728/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764192 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764167.740671:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764167.740676:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764167.740677:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:3.0:1787764167.740680:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764167.740682:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:3.0:1787764167.740686:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:3.0:1787764167.740692:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:3.0:1787764167.740700:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764167.740706:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764167.740710:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764167.740713:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:3.0:1787764167.740715:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764167.740721:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00000040:2.0:1787764167.740722:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764167.740728:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00080000:3.0:1787764167.740729:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764167.740734:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764167.740735:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832697728, offset 0
02000000:00000001:3.0:1787764167.740736:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764167.740740:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764167.740743:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764167.740745:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787764167.740747:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:2.0:1787764167.740749:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000008:00000001:3.0:1787764167.740750:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764167.740753:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764167.740757:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc000.
00000020:00000040:3.0:1787764167.740759:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000400:00000200:2.0:1787764167.740761:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:3.0:1787764167.740763:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764167.740766:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764167.740768:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764167.740769:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764167.740771:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764167.740774:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
02000000:00000010:3.0:1787764167.740775:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8613c00.
00000100:00000001:3.0:1787764167.740781:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764167.740784:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:3.0:1787764167.740787:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc000 x1874599832697984/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787764167.740794:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:3.0:1787764167.740796:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764167.740799:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764167.740803:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000040:3.0:1787764167.740804:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f3fc1c6c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc000 x1874599832697984/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764167.740813:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:3.0:1787764167.740815:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764167.740819:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787764167.740825:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58686:58686:256:4294967295] 192.168.203.110@tcp LPNI seq info [58686:58686:8:4294967295]
00000100:00000040:3.0:1787764167.740825:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:3.0:1787764167.740830:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764167.740836:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787764167.740840:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:2.0:1787764167.740841:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:3.0:1787764167.740843:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764167.740848:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764167.740855:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21882
00000400:00000200:2.0:1787764167.740861:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764167.740870:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764167.740875:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000010:2.0:1787764167.740880:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000400:00000200:3.0:1787764167.740882:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764167.740886:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764167.740889:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764167.740893:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1787764167.740895:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:3.0:1787764167.740897:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764167.740902:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764167.740904:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:2.0:1787764167.740905:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000020:00000040:3.0:1787764167.740905:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:3.0:1787764167.740908:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764167.740911:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:3.0:1787764167.740914:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:3.0:1787764167.740917:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000800:00000200:2.1:1787764167.740921:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:3.0:1787764167.740923:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764167.740926:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764167.740929:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000200:2.1:1787764167.740931:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:3.0:1787764167.740932:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764167.740935:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:3.0:1787764167.740940:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764167.740942:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764167.740945:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000100:2.0:1787764167.740947:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
02000000:00000001:3.0:1787764167.740948:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000010:2.0:1787764167.740951:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
02000000:00000001:3.0:1787764167.740952:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764167.740956:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787764167.740960:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1787764167.740961:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764167.740962:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:3.0:1787764167.740963:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764167.740969:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bed00.
00000020:00000040:3.0:1787764167.740973:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000400:00000200:2.0:1787764167.740974:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:3.0:1787764167.740976:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764167.740979:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764167.740981:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764167.740983:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1787764167.740984:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
02000000:00000001:3.0:1787764167.740984:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764167.740991:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
02000000:00000010:3.0:1787764167.740991:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610400.
00000100:00000001:3.0:1787764167.740994:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764167.740999:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00100000:3.0:1787764167.741001:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bed00 x1874599832698112/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:2.0:1787764167.741009:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bf480 x1874599832697728/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764192 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764167.741011:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764167.741014:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764167.741021:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000079751c9a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bed00 x1874599832698112/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764167.741037:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764167.741041:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764167.741041:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21892)
00000100:00000040:2.0:1787764167.741051:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bf480 x1874599832697728/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764192 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.741079:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764167.741082:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764167.741089:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764167.741094:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764167.741097:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316556019
00000100:00000010:2.0:1787764167.741103:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7720.
00000100:00000001:2.0:1787764167.741110:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787764167.741118:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316556019
00000100:00000010:2.0:1787764167.741121:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb77e0.
00000100:00000001:2.0:1787764167.741125:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787764167.741128:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764167.741131:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764167.741156:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764167.741161:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764167/real 1787764167]  req@ffff9751f87bf480 x1874599832697728/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764167.741171:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764167.741174:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764167.741176:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764167.741179:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764167.741181:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764167.741184:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764167.741188:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bf480 x1874599832697728/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764167.741198:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bf480 x1874599832697728/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.741206:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764167.741208:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.741210:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764167.741212:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764167.741217:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bf480 x1874599832697728/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.741226:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764167.741231:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bf480 x1874599832697728/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.741239:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764167.741243:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.741245:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764167.741246:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.741249:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764167.741251:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764167.741255:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764167.741259:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764167.741261:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764167.741267:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bf480 x1874599832697728/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764167.741276:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832697728:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764167.741282:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.741284:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764167.741288:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bf480 x1874599832697728/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.741296:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764167.741299:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764167.741302:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764167.741306:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764167.741308:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764167.741310:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764167.741312:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764167.741315:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613800.
02000000:00000001:2.0:1787764167.741319:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764167.741322:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764167.741324:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bf480.
00000100:00000001:2.0:1787764167.741328:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764167.741329:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.741332:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.741338:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764167.741340:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764167.741342:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764167.741344:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764167.741346:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764167.741350:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87be1c0 x1874599832697856/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.741358:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764167.741360:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764167.741362:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764167.741364:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764167.741367:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764167.741371:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832697856:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764167.741374:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764167.741377:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764167.741379:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764167.741381:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764167.741385:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764167.741387:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764167.741391:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764167.741394:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764167.741398:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764167.741402:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832697856, portal 10
00000100:00000001:2.0:1787764167.741404:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764167.741406:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049854400 : -115096659697216 : ffff9751f87be1c0)
00000100:00000040:2.0:1787764167.741414:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87be1c0 x1874599832697856/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764192 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.741422:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764167.741424:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764167.741427:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764167.741430:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832697856, offset 0
00000400:00000200:2.0:1787764167.741436:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764167.741441:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764167.741445:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764167.741449:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764167.741452:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764167.741455:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764167.741457:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764167.741461:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58687:58687:256:4294967295] 192.168.203.110@tcp LPNI seq info [58687:58687:8:4294967295]
00000400:00000200:2.0:1787764167.741466:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764167.741473:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764167.741479:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764167.741482:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764167.741487:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764167.741490:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787764167.741495:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764167.741498:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1787764167.741502:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764167.741504:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764167.741508:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764167.741511:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764167.741515:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764167.741518:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764167.741520:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764167.741524:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87be1c0 x1874599832697856/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764192 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.741533:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764167.741538:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87be1c0 x1874599832697856/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764192 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.741546:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764167.741548:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764167.741552:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764167.741556:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764167.741560:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764167.741562:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316554784
00000100:00000001:2.0:1787764167.741565:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316556019
00000100:00000001:2.0:1787764167.741567:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316554784
00000100:00000001:2.0:1787764167.741570:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316556019
00000100:00000001:2.0:1787764167.741572:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764167.741575:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764167.741577:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764167.741581:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764167/real 1787764167]  req@ffff9751f87be1c0 x1874599832697856/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764167.741589:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764167.741591:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764167.741593:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764167.741595:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764167.741597:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764167.741600:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764167.741603:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87be1c0 x1874599832697856/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764167.741612:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87be1c0 x1874599832697856/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.741619:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764167.741621:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.741623:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764167.741624:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764167.741628:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87be1c0 x1874599832697856/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.741636:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764167.741641:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87be1c0 x1874599832697856/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.741650:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764167.741652:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.741654:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764167.741656:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.741658:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764167.741670:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764167.741673:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764167.741677:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764167.741679:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764167.741684:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87be1c0 x1874599832697856/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764167.741693:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87be1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832697856:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764167.741697:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764167.741700:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc000 x1874599832697984/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.741707:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764167.741709:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764167.741711:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764167.741713:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764167.741716:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764167.741719:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832697984:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764167.741723:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764167.741725:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764167.741727:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764167.741729:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764167.741733:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764167.741735:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764167.741739:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764167.741742:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764167.741745:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764167.741748:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832697984, portal 10
00000100:00000001:2.0:1787764167.741750:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764167.741752:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049845760 : -115096659705856 : ffff9751f87bc000)
00000100:00000040:2.0:1787764167.741758:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc000 x1874599832697984/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764192 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.741766:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764167.741769:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764167.741771:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764167.741773:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832697984, offset 0
00000400:00000200:2.0:1787764167.741778:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764167.741782:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764167.741786:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764167.741789:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764167.741792:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764167.741795:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764167.741798:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764167.741801:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58688:58688:256:4294967295] 192.168.203.110@tcp LPNI seq info [58688:58688:8:4294967295]
00000400:00000200:2.0:1787764167.741806:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764167.741812:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764167.741817:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764167.741821:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764167.741825:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764167.741828:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787764167.741832:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764167.741835:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1787764167.741840:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764167.741842:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764167.741846:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764167.741850:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764167.741853:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764167.741856:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764167.741858:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764167.741863:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc000 x1874599832697984/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764192 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.741872:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764167.741875:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc000 x1874599832697984/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764192 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.741883:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764167.741885:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764167.741888:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764167.741891:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764167.741895:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764167.741898:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316554772
00000100:00000001:2.0:1787764167.741900:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316556019
00000100:00000001:2.0:1787764167.741903:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316554772
00000100:00000001:2.0:1787764167.741905:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316556019
00000100:00000001:2.0:1787764167.741908:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764167.741910:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764167.741912:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764167.741917:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764167/real 1787764167]  req@ffff9751f87bc000 x1874599832697984/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764167.741926:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764167.741927:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764167.741929:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764167.741932:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764167.741934:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764167.741936:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764167.741940:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc000 x1874599832697984/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764167.741949:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc000 x1874599832697984/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.741958:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764167.741960:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.741962:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764167.741963:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764167.741967:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc000 x1874599832697984/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.741975:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764167.741980:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc000 x1874599832697984/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.741988:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764167.741991:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.741992:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764167.741994:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.741997:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764167.741998:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764167.742002:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764167.742005:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764167.742007:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764167.742012:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc000 x1874599832697984/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764167.742022:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832697984:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764167.742026:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764167.742030:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bed00 x1874599832698112/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.742037:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764167.742038:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764167.742041:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764167.742043:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764167.742046:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764167.742049:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832698112:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764167.742053:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764167.742055:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764167.742057:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764167.742059:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764167.742062:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764167.742065:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764167.742085:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764167.742088:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764167.742093:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764167.742096:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832698112, portal 4
00000100:00000001:2.0:1787764167.742098:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764167.742099:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049857280 : -115096659694336 : ffff9751f87bed00)
00000100:00000040:2.0:1787764167.742106:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bed00 x1874599832698112/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764192 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.742113:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764167.742116:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764167.742118:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764167.742121:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832698112, offset 0
00000400:00000200:2.0:1787764167.742125:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764167.742129:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764167.742152:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764167.742156:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764167.742159:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764167.742162:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764167.742165:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764167.742168:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58689:58689:256:4294967295] 192.168.203.110@tcp LPNI seq info [58689:58689:8:4294967295]
00000400:00000200:2.0:1787764167.742173:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764167.742180:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764167.742185:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764167.742189:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764167.742194:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764167.742198:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:2.1:1787764167.742203:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764167.742206:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:2.0:1787764167.742211:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764167.742214:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764167.742217:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764167.742221:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764167.742225:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764167.742227:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764167.742230:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764167.742234:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bed00 x1874599832698112/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764192 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.742243:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764167.742247:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bed00 x1874599832698112/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764192 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.742254:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764167.742256:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764167.742259:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764167.742263:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764167.742266:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764167.742269:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316554790
00000100:00000001:2.0:1787764167.742271:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316556019
00000100:00000001:2.0:1787764167.742273:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316554790
00000100:00000001:2.0:1787764167.742276:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316556019
00000100:00000001:2.0:1787764167.742278:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764167.742280:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764167.742282:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764167.742286:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764167/real 1787764167]  req@ffff9751f87bed00 x1874599832698112/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764167.742295:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764167.742297:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764167.742299:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764167.742301:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764167.742303:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764167.742305:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764167.742309:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bed00 x1874599832698112/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764167.742320:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bed00 x1874599832698112/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.742328:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764167.742329:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.742331:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764167.742333:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764167.742337:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bed00 x1874599832698112/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.742345:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764167.742349:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bed00 x1874599832698112/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.742357:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764167.742359:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.742361:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764167.742363:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.742365:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764167.742367:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764167.742369:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764167.742373:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764167.742375:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764167.742380:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bed00 x1874599832698112/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764167.742390:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832698112:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764167.742394:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.742396:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764167.742399:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87be1c0 x1874599832697856/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.742408:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764167.742409:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764167.742412:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764167.742415:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764167.742417:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764167.742419:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764167.742421:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764167.742423:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611800.
02000000:00000001:2.0:1787764167.742426:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764167.742428:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764167.742430:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87be1c0.
00000100:00000001:2.0:1787764167.742432:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764167.742434:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.742436:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764167.742439:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc000 x1874599832697984/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.742446:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764167.742448:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764167.742450:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764167.742453:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764167.742454:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764167.742456:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764167.742458:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764167.742459:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8613c00.
02000000:00000001:2.0:1787764167.742462:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764167.742463:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764167.742465:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc000.
00000100:00000001:2.0:1787764167.742468:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764167.742469:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.742471:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764167.742474:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bed00 x1874599832698112/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764192 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764167.742481:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764167.742483:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764167.742485:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764167.742488:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764167.742489:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764167.742491:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764167.742492:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764167.742494:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610400.
02000000:00000001:2.0:1787764167.742496:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764167.742498:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764167.742499:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bed00.
00000100:00000001:2.0:1787764167.742502:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764167.742503:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.742506:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764167.742509:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764167.742510:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764167.742513:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764167.742515:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764168.380233:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764168.380239:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764168.380240:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764168.380242:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764168.380245:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764168.380247:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764168.380252:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764168.380253:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764168.380255:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764168.380257:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764168.380259:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764168.380262:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764168.380264:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764168.380265:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764168.380267:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764168.380308:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764168.380311:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764168.380314:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787764168.764214:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787764168.764220:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764168.764222:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764168.764227:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787764168.764230:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764168.764233:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764168.764235:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764168.764237:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764168.764240:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764169.020214:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316556019, at 4316556032
00000100:00000010:3.0:1787764169.020228:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb77e0.
00000100:00000040:2.0:1787764169.020250:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316556019, at 4316556032
00000100:00000010:2.0:1787764169.020258:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7720.
00000100:00000001:2.0:1787764169.404254:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764169.404256:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764169.404265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764169.404266:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764169.404270:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764169.404275:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764169.404277:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764169.404279:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764169.404287:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764169.404288:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764169.404292:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764169.404297:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764169.404301:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764169.404304:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764169.404308:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764169.404351:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764169.404356:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764169.404360:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787764169.788272:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787764169.788282:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764169.788284:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764169.788291:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787764169.788299:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764169.788307:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764169.788311:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764169.788314:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764169.788316:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764170.428272:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764170.428278:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764170.428279:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764170.428292:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764170.428292:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764170.428300:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764170.428301:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764170.428304:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764170.428308:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764170.428312:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764170.428320:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764170.428325:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764170.428328:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764170.428332:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764170.428337:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764170.428379:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764170.428385:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764170.428389:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787764170.812252:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787764170.812260:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764170.812262:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764170.812267:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787764170.812271:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764170.812275:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764170.812278:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764170.812281:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764170.812283:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764171.452247:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764171.452250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764171.452255:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764171.452256:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764171.452258:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764171.452261:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764171.452263:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764171.452267:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764171.452267:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764171.452269:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764171.452271:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764171.452274:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764171.452278:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764171.452281:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764171.452284:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764171.452306:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764171.452311:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764171.452315:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787764171.836249:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787764171.836258:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764171.836261:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764171.836267:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787764171.836270:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764171.836273:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764171.836275:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764171.836278:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764171.836280:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764172.476257:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764172.476261:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764172.476263:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764172.476265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764172.476266:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764172.476271:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764172.476272:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764172.476275:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764172.476277:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764172.476279:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764172.476281:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764172.476283:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764172.476285:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764172.476286:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764172.476288:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764172.476290:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764172.476291:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764172.476294:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764172.476297:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764172.476298:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764172.476300:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764172.476352:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764172.476356:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764172.476358:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:3.0:1787764172.860234:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764172.860244:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764172.860246:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764172.860250:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764172.860255:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764172.860262:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21887
00000400:00000200:3.0:1787764172.860270:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764172.860279:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764172.860282:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764172.860285:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764172.860287:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764172.860288:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764172.860293:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764172.860297:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787764172.860298:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000020:00000001:3.0:1787764172.860304:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:1.0:1787764172.860305:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:3.0:1787764172.860306:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:3.0:1787764172.860311:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00100000:1.0:1787764172.860317:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000020:00000040:3.0:1787764172.860317:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:3.0:1787764172.860327:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000001:1.0:1787764172.860330:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1787764172.860330:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000001:1.0:1787764172.860337:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764172.860337:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764172.860339:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764172.860345:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764172.860346:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764172.860347:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764172.860351:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00000001:1.0:1787764172.860353:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1787764172.860356:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764172.860360:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764172.860362:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764172.860363:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764172.860367:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764172.860370:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764172.860372:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764172.860374:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764172.860382:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bde00.
00000020:00000040:3.0:1787764172.860384:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787764172.860387:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764172.860389:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764172.860392:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764172.860394:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764172.860397:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764172.860405:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610c00.
00000100:00000001:3.0:1787764172.860413:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764172.860424:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bde00 x1874599832698240/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764172.860435:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764172.860439:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764172.860445:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000941163c2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bde00 x1874599832698240/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764172.860466:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764172.860468:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764172.860471:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:3.0:1787764172.860472:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764172.860473:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764172.860475:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764172.860477:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21887
00000400:00000200:3.0:1787764172.860481:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764172.860484:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764172.860486:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764172.860489:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764172.860491:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764172.860493:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:3.0:1787764172.860493:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764172.860497:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764172.860499:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:3.0:1787764172.860500:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764172.860503:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00000001:2.0:1787764172.860504:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:3.0:1787764172.860505:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:3.0:1787764172.860509:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:3.0:1787764172.860512:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:2.0:1787764172.860513:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bde00 x1874599832698240/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787764172.860514:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:3.0:1787764172.860523:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764172.860529:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764172.860530:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764172.860532:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764172.860536:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:3.0:1787764172.860538:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764172.860539:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764172.860544:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764172.860544:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00100000:2.0:1787764172.860550:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832698240:192.168.203.110@tcp:250:kworker.0
00000020:00000040:3.0:1787764172.860551:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00000001:2.0:1787764172.860555:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764172.860560:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:3.0:1787764172.860560:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:2.0:1787764172.860563:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764172.860567:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:3.0:1787764172.860568:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764172.860572:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:2.0:1787764172.860573:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764172.860576:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764172.860578:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764172.860582:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764172.860583:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764172.860586:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:2.0:1787764172.860588:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000008:00100000:3.0:1787764172.860591:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000010:2.0:1787764172.860595:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000008:00000001:3.0:1787764172.860595:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764172.860600:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764172.860604:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832698240, portal 25
00000100:00000010:3.0:1787764172.860605:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bfc00.
00000100:00000001:2.0:1787764172.860607:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764172.860610:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049853440 : -115096659698176 : ffff9751f87bde00)
00000020:00000040:3.0:1787764172.860610:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787764172.860617:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:2.0:1787764172.860619:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bde00 x1874599832698240/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787764172.860621:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764172.860627:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1787764172.860631:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:3.0:1787764172.860632:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:2.0:1787764172.860636:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
02000000:00000001:3.0:1787764172.860637:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764172.860639:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764172.860643:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832698240, offset 0
02000000:00000010:3.0:1787764172.860646:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8610800.
00000400:00000200:2.0:1787764172.860650:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:3.0:1787764172.860654:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764172.860659:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00100000:3.0:1787764172.860664:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bfc00 x1874599832698368/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787764172.860668:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764172.860677:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:3.0:1787764172.860678:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764172.860681:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764172.860683:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000040:3.0:1787764172.860687:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003c1f6b31] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bfc00 x1874599832698368/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764172.860689:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764172.860695:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764172.860702:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58690:58690:256:4294967295] 192.168.203.110@tcp LPNI seq info [58690:58690:8:4294967295]
00000100:00000001:3.0:1787764172.860705:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764172.860708:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:3.0:1787764172.860710:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764172.860713:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:3.0:1787764172.860716:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:2.0:1787764172.860720:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00080000:3.0:1787764172.860720:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764172.860724:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:3.0:1787764172.860728:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21887
00000800:00000200:2.0:1787764172.860731:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764172.860736:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000010:2.0:1787764172.860739:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000400:00000200:3.0:1787764172.860741:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:3.0:1787764172.860744:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000200:2.0:1787764172.860748:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:3.0:1787764172.860749:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:2.0:1787764172.860753:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:3.0:1787764172.860753:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764172.860759:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764172.860763:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:2.1:1787764172.860763:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:3.0:1787764172.860766:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:2.1:1787764172.860768:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000020:00000040:3.0:1787764172.860769:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:3.0:1787764172.860772:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000100:2.0:1787764172.860776:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000020:00000040:3.0:1787764172.860776:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000010:2.0:1787764172.860778:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764172.860784:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764172.860788:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:3.0:1787764172.860789:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:2.0:1787764172.860792:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000020:00000001:3.0:1787764172.860793:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000010:2.0:1787764172.860795:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764172.860802:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:3.0:1787764172.860803:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000200:2.0:1787764172.860807:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bde00 x1874599832698240/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764172.860809:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764172.860816:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764172.860818:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:3.0:1787764172.860823:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:2.0:1787764172.860824:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bde00 x1874599832698240/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764197 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:3.0:1787764172.860830:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00000001:2.0:1787764172.860836:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764172.860838:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764172.860839:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764172.860843:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764172.860845:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787764172.860846:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764172.860848:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1787764172.860853:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:3.0:1787764172.860853:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764172.860856:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316561139
00000100:00000001:3.0:1787764172.860857:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:3.0:1787764172.860860:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000010:2.0:1787764172.860863:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7720.
00000008:00000001:3.0:1787764172.860864:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764172.860869:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764172.860872:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000010:3.0:1787764172.860873:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc3c0.
00000020:00000040:3.0:1787764172.860877:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787764172.860879:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1787764172.860880:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316561139
02000000:00000001:3.0:1787764172.860881:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764172.860885:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000010:2.0:1787764172.860887:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7ae0.
02000000:00000001:3.0:1787764172.860887:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764172.860891:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764172.860892:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787764172.860895:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000010:3.0:1787764172.860896:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8612000.
00000100:00000001:2.0:1787764172.860898:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764172.860902:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:3.0:1787764172.860902:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764172.860907:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764172/real 1787764172]  req@ffff9751f87bde00 x1874599832698240/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764172.860910:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc3c0 x1874599832698496/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000001:2.0:1787764172.860927:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:3.0:1787764172.860929:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000001:2.0:1787764172.860930:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764172.860933:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000100:00000001:3.0:1787764172.860934:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764172.860937:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000100:00000040:3.0:1787764172.860940:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000060bf8470] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc3c0 x1874599832698496/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1787764172.860941:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764172.860945:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764172.860951:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bde00 x1874599832698240/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764172.860952:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764172.860958:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1787764172.860966:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000200:2.0:1787764172.860969:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bde00 x1874599832698240/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787764172.860970:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:3.0:1787764172.860979:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787764172.860982:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787764172.860984:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00080000:3.0:1787764172.860985:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764172.860987:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764172.860988:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787764172.860990:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764172.860993:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1787764172.860993:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21887
00000100:00100000:2.0:1787764172.860998:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bde00 x1874599832698240/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764172.861000:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764172.861007:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787764172.861012:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:3.0:1787764172.861012:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764172.861018:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000040:2.0:1787764172.861020:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bde00 x1874599832698240/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764172.861021:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764172.861027:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764172.861048:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764172.861052:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000001:3.0:1787764172.861052:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:3.0:1787764172.861055:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000100:00000001:2.0:1787764172.861057:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:3.0:1787764172.861058:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1787764172.861062:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000040:3.0:1787764172.861062:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000001:2.0:1787764172.861065:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:3.0:1787764172.861066:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:2.0:1787764172.861069:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000020:00000001:3.0:1787764172.861070:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:2.0:1787764172.861072:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:3.0:1787764172.861075:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00080000:2.0:1787764172.861077:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:3.0:1787764172.861079:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00080000:2.0:1787764172.861084:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000040:3.0:1787764172.861084:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764172.861090:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:3.0:1787764172.861091:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:3.0:1787764172.861097:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000040:2.0:1787764172.861100:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bde00 x1874599832698240/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:3.0:1787764172.861103:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764172.861109:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764172.861113:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:2.0:1787764172.861117:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bde00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832698240:192.168.203.110@tcp:250:kworker.0
02000000:00000001:3.0:1787764172.861117:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764172.861124:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787764172.861124:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764172.861126:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764172.861128:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000008:00100000:3.0:1787764172.861130:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:3.0:1787764172.861134:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764172.861152:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bde00 x1874599832698240/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861159:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
00000100:00000001:3.0:1787764172.861160:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764172.861163:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000010:3.0:1787764172.861165:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f87bc780.
02000000:00000010:2.0:1787764172.861166:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
00000020:00000040:3.0:1787764172.861170:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:2.0:1787764172.861171:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764172.861175:0:257575:0:(genops.c:1217:class_import_put()) Process entered
02000000:00000001:3.0:1787764172.861176:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000020:00000040:2.0:1787764172.861177:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
02000000:00000001:3.0:1787764172.861179:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764172.861180:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764172.861184:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610c00.
02000000:00000001:2.0:1787764172.861187:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764172.861189:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764172.861190:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bde00.
00000100:00000001:3.0:1787764172.861190:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764172.861193:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787764172.861195:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764172.861197:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:3.0:1787764172.861198:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764172.861199:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
02000000:00000010:3.0:1787764172.861207:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f8611c00.
00000100:00000001:2.0:1787764172.861209:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764172.861211:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764172.861212:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764172.861217:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764172.861219:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764172.861222:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00100000:3.0:1787764172.861222:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f87bc780 x1874599832698624/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:2.0:1787764172.861228:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bfc00 x1874599832698368/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764172.861244:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764172.861248:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764172.861249:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764172.861253:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764172.861254:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7d56184] to pc [ptlrpcd_rcv+-1]  req@ffff9751f87bc780 x1874599832698624/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764172.861257:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764172.861261:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764172.861265:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764172.861266:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764172.861271:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21897)
00000100:00100000:2.0:1787764172.861275:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832698368:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764172.861281:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764172.861285:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764172.861287:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764172.861289:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764172.861293:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764172.861295:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764172.861298:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764172.861300:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764172.861303:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764172.861305:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832698368, portal 10
00000100:00000001:2.0:1787764172.861306:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764172.861308:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049861120 : -115096659690496 : ffff9751f87bfc00)
00000100:00000040:2.0:1787764172.861313:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bfc00 x1874599832698368/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861317:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764172.861319:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764172.861320:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764172.861322:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832698368, offset 0
00000400:00000200:2.0:1787764172.861324:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764172.861327:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764172.861329:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764172.861332:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764172.861333:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764172.861335:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764172.861337:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764172.861339:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58691:58691:256:4294967295] 192.168.203.110@tcp LPNI seq info [58691:58691:8:4294967295]
00000400:00000200:2.0:1787764172.861341:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764172.861346:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764172.861348:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764172.861351:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764172.861354:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764172.861356:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787764172.861359:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764172.861361:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1787764172.861363:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764172.861364:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764172.861366:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764172.861369:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764172.861371:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764172.861373:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764172.861375:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764172.861379:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bfc00 x1874599832698368/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861387:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764172.861390:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bfc00 x1874599832698368/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764197 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861397:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764172.861398:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764172.861402:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764172.861406:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764172.861409:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764172.861412:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316559904
00000100:00000001:2.0:1787764172.861415:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316561139
00000100:00000001:2.0:1787764172.861417:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316559904
00000100:00000001:2.0:1787764172.861419:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316561139
00000100:00000001:2.0:1787764172.861422:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764172.861424:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764172.861426:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764172.861430:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764172/real 1787764172]  req@ffff9751f87bfc00 x1874599832698368/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764172.861441:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764172.861442:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764172.861443:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764172.861445:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764172.861446:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764172.861447:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764172.861449:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bfc00 x1874599832698368/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764172.861455:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bfc00 x1874599832698368/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861459:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764172.861460:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764172.861461:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764172.861462:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764172.861464:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bfc00 x1874599832698368/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861468:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764172.861471:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bfc00 x1874599832698368/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861476:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764172.861477:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764172.861478:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764172.861479:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764172.861480:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764172.861490:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764172.861492:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764172.861494:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764172.861495:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764172.861498:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bfc00 x1874599832698368/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764172.861503:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bfc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832698368:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764172.861506:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764172.861508:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc3c0 x1874599832698496/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861512:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764172.861513:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764172.861514:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764172.861515:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764172.861516:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764172.861518:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832698496:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764172.861520:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764172.861521:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764172.861522:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764172.861523:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764172.861526:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764172.861527:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764172.861530:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764172.861531:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764172.861533:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764172.861535:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832698496, portal 10
00000100:00000001:2.0:1787764172.861537:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764172.861538:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049846720 : -115096659704896 : ffff9751f87bc3c0)
00000100:00000040:2.0:1787764172.861542:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc3c0 x1874599832698496/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861546:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764172.861547:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764172.861549:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764172.861550:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832698496, offset 0
00000400:00000200:2.0:1787764172.861553:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764172.861556:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764172.861558:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764172.861560:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764172.861562:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764172.861564:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764172.861565:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764172.861567:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58692:58692:256:4294967295] 192.168.203.110@tcp LPNI seq info [58692:58692:8:4294967295]
00000400:00000200:2.0:1787764172.861570:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764172.861574:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764172.861577:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764172.861579:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764172.861582:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764172.861583:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787764172.861586:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764172.861588:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1787764172.861590:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764172.861592:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764172.861594:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764172.861596:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764172.861598:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764172.861599:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764172.861601:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764172.861603:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc3c0 x1874599832698496/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861608:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764172.861610:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc3c0 x1874599832698496/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764197 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861614:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764172.861615:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764172.861617:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764172.861619:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764172.861621:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764172.861622:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316559892
00000100:00000001:2.0:1787764172.861624:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316561139
00000100:00000001:2.0:1787764172.861625:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316559892
00000100:00000001:2.0:1787764172.861627:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316561139
00000100:00000001:2.0:1787764172.861628:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764172.861629:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764172.861630:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764172.861632:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764172/real 1787764172]  req@ffff9751f87bc3c0 x1874599832698496/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764172.861637:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764172.861637:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764172.861638:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764172.861640:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764172.861641:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764172.861642:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764172.861644:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc3c0 x1874599832698496/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764172.861649:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc3c0 x1874599832698496/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861653:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764172.861654:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764172.861655:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764172.861656:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764172.861658:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc3c0 x1874599832698496/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861662:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764172.861664:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc3c0 x1874599832698496/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861668:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764172.861670:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764172.861671:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764172.861672:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764172.861673:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764172.861674:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764172.861676:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764172.861678:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764172.861679:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764172.861681:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc3c0 x1874599832698496/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764172.861686:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832698496:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764172.861688:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764172.861690:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764172.861691:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bfc00 x1874599832698368/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861695:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764172.861696:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764172.861697:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5ba000.
02000000:00000001:2.0:1787764172.861699:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764172.861700:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764172.861701:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764172.861702:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764172.861703:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8610800.
02000000:00000001:2.0:1787764172.861705:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764172.861706:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764172.861707:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bfc00.
00000100:00000001:2.0:1787764172.861708:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764172.861709:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764172.861711:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764172.861713:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc3c0 x1874599832698496/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861721:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764172.861722:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764172.861723:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764172.861724:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764172.861725:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764172.861726:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764172.861727:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764172.861728:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8612000.
02000000:00000001:2.0:1787764172.861729:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764172.861729:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764172.861730:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc3c0.
00000100:00000001:2.0:1787764172.861731:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764172.861732:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764172.861733:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764172.861736:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764172.861736:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764172.861738:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764172.861739:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764172.861739:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764172.861741:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f87bc780 x1874599832698624/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861745:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764172.861746:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764172.861747:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764172.861748:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764172.861750:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764172.861751:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832698624:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764172.861754:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764172.861755:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764172.861756:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764172.861757:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764172.861759:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764172.861760:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764172.861763:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764172.861764:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764172.861766:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764172.861768:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832698624, portal 4
00000100:00000001:2.0:1787764172.861769:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764172.861770:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049847680 : -115096659703936 : ffff9751f87bc780)
00000100:00000040:2.0:1787764172.861774:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f87bc780 x1874599832698624/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861778:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764172.861780:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764172.861781:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764172.861783:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832698624, offset 0
00000400:00000200:2.0:1787764172.861785:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764172.861787:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764172.861789:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764172.861790:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764172.861792:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764172.861794:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764172.861795:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764172.861797:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58693:58693:256:4294967295] 192.168.203.110@tcp LPNI seq info [58693:58693:8:4294967295]
00000400:00000200:2.0:1787764172.861800:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764172.861804:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764172.861807:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764172.861809:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764172.861811:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764172.861813:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:2.1:1787764172.861816:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764172.861818:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:2.0:1787764172.861820:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764172.861821:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764172.861823:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764172.861825:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764172.861827:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764172.861828:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764172.861830:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764172.861832:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f87bc780 x1874599832698624/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861838:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764172.861839:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f87bc780 x1874599832698624/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764197 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861843:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764172.861845:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764172.861846:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764172.861848:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764172.861850:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764172.861852:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316559909
00000100:00000001:2.0:1787764172.861853:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316561139
00000100:00000001:2.0:1787764172.861855:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316559909
00000100:00000001:2.0:1787764172.861857:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316561139
00000100:00000001:2.0:1787764172.861858:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764172.861859:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764172.861860:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764172.861862:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764172/real 1787764172]  req@ffff9751f87bc780 x1874599832698624/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764172.861867:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764172.861868:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764172.861869:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764172.861871:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764172.861872:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764172.861873:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764172.861875:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f87bc780 x1874599832698624/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764172.861880:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f87bc780 x1874599832698624/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861884:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764172.861885:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764172.861886:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764172.861887:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764172.861889:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f87bc780 x1874599832698624/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861893:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764172.861895:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f87bc780 x1874599832698624/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861899:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764172.861901:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764172.861902:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764172.861902:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764172.861903:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764172.861904:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764172.861906:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764172.861908:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764172.861909:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764172.861911:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f87bc780 x1874599832698624/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764172.861916:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f87bc780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832698624:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764172.861918:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764172.861919:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764172.861920:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f87bc780 x1874599832698624/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764197 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764172.861925:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764172.861926:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764172.861927:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bb800.
02000000:00000001:2.0:1787764172.861929:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764172.861930:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764172.861931:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764172.861932:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764172.861933:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f8611c00.
02000000:00000001:2.0:1787764172.861934:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764172.861935:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764172.861936:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f87bc780.
00000100:00000001:2.0:1787764172.861937:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764172.861938:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764172.861939:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764172.861941:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764172.861942:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764172.861943:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764172.861944:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764173.500279:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764173.500286:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764173.500288:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764173.500291:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764173.500294:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764173.500299:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764173.500304:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764173.500305:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764173.500311:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764173.500312:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764173.500324:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764173.500327:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764173.500328:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764173.500334:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764173.500335:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764173.500400:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764173.500405:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764173.500410:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764173.884280:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764173.884286:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764173.884288:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764173.884293:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764173.884296:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764173.884299:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764173.884301:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764173.884303:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764173.884306:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764174.140231:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316561139, at 4316561152
00000100:00000010:3.0:1787764174.140244:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7ae0.
00000100:00000040:2.0:1787764174.140302:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316561139, at 4316561152
00000100:00000010:2.0:1787764174.140315:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7720.
00000100:00000001:2.0:1787764174.524224:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764174.524232:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764174.524232:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764174.524235:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764174.524238:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764174.524241:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764174.524245:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764174.524246:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764174.524250:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764174.524251:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764174.524256:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764174.524258:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764174.524259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764174.524262:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764174.524262:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764174.524305:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764174.524309:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764174.524311:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787764174.908199:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787764174.908205:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764174.908208:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764174.908212:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787764174.908215:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764174.908218:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764174.908220:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764174.908222:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764174.908224:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764175.548193:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764175.548201:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764175.548204:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764175.548205:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764175.548210:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764175.548210:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764175.548212:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764175.548218:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00100000:2.0:1787764175.548219:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764175.548223:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764175.548224:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764175.548227:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764175.548229:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764175.548231:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764175.548234:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764175.548234:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764175.548241:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764175.548244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764175.548247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764175.548250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764175.548252:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764175.548287:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764175.548291:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764175.548294:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787764175.932236:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787764175.932243:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764175.932245:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764175.932249:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787764175.932252:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764175.932255:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764175.932257:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764175.932259:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764175.932261:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764176.572231:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764176.572237:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764176.572239:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764176.572241:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764176.572245:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764176.572246:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764176.572252:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764176.572252:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764176.572255:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764176.572255:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764176.572261:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764176.572263:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764176.572264:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764176.572265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764176.572269:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764176.572300:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764176.572304:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764176.572306:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787764176.956229:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787764176.956237:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764176.956240:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764176.956245:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787764176.956250:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764176.956254:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764176.956257:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764176.956261:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764176.956265:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764177.596230:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764177.596236:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764177.596237:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764177.596242:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764177.596243:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764177.596249:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764177.596253:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764177.596254:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764177.596258:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764177.596259:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764177.596261:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764177.596263:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764177.596264:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.596268:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764177.596273:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764177.596278:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764177.596281:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764177.596283:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764177.596287:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764177.596291:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764177.596293:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764177.596331:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764177.596335:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764177.596338:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787764177.980240:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787764177.980248:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764177.980250:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764177.980253:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:0.0:1787764177.980255:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787764177.980261:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:1.0:1787764177.980262:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787764177.980266:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764177.980266:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:0.0:1787764177.980270:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:1.0:1787764177.980270:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764177.980272:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764177.980275:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:0.0:1787764177.980276:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764177.980282:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21892
00000400:00000200:1.0:1787764177.980288:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764177.980294:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764177.980296:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764177.980299:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764177.980301:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764177.980303:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764177.980304:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764177.980307:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764177.980308:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787764177.980311:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764177.980314:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787764177.980315:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787764177.980317:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787764177.980319:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764177.980320:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764177.980322:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764177.980323:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764177.980325:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787764177.980327:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764177.980329:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764177.980331:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764177.980333:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764177.980335:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764177.980337:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764177.980338:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764177.980353:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930780.
00000020:00000040:1.0:1787764177.980356:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787764177.980359:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764177.980360:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764177.980362:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764177.980364:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764177.980367:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764177.980374:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8400.
00000100:00000001:1.0:1787764177.980379:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764177.980386:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930780 x1874599832698752/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764177.980394:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764177.980398:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764177.980401:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3977950] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930780 x1874599832698752/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764177.980416:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764177.980419:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764177.980421:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787764177.980422:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764177.980424:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764177.980425:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764177.980427:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21892
00000400:00000200:1.0:1787764177.980430:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764177.980432:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764177.980434:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764177.980436:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764177.980438:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764177.980440:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764177.980442:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764177.980444:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764177.980445:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787764177.980446:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1787764177.980447:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:1.0:1787764177.980448:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787764177.980452:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787764177.980455:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787764177.980455:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1787764177.980460:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764177.980460:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1787764177.980465:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764177.980466:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764177.980468:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764177.980471:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00000040:2.0:1787764177.980473:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930780 x1874599832698752/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787764177.980474:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764177.980479:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764177.980481:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764177.980487:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764177.980490:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764177.980491:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764177.980496:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1787764177.980498:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00000001:1.0:1787764177.980500:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764177.980502:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764177.980505:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764177.980508:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000010:1.0:1787764177.980510:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932580.
02000000:00000001:2.0:1787764177.980512:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000040:1.0:1787764177.980516:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764177.980517:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764177.980523:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00100000:2.0:1787764177.980526:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832698752:192.168.203.110@tcp:250:kworker.0
02000000:00000001:1.0:1787764177.980527:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764177.980532:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764177.980535:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787764177.980536:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764177.980538:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764177.980542:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000010:1.0:1787764177.980546:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8800.
02000000:00000001:2.0:1787764177.980546:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764177.980553:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764177.980553:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764177.980559:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
00000100:00100000:1.0:1787764177.980560:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932580 x1874599832698880/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1787764177.980565:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764177.980572:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764177.980576:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764177.980577:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:1.0:1787764177.980581:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d026924] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932580 x1874599832698880/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1787764177.980584:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000100:00000001:1.0:1787764177.980592:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764177.980594:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:2.0:1787764177.980595:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00080000:1.0:1787764177.980598:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787764177.980601:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764177.980602:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764177.980604:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:2.0:1787764177.980606:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832698752, portal 25
00000100:00080000:1.0:1787764177.980607:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21892
00000100:00000001:2.0:1787764177.980613:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000400:00000200:1.0:1787764177.980614:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764177.980617:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764177.980618:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764177.980620:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764177.980621:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764177.980623:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764177.980626:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764177.980626:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497723264 : -115097211828352 : ffff9751d7930780)
00000020:00000001:1.0:1787764177.980632:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764177.980634:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787764177.980636:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764177.980638:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787764177.980641:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:2.0:1787764177.980641:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930780 x1874599832698752/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1787764177.980645:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787764177.980651:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764177.980653:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764177.980654:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764177.980655:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764177.980656:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787764177.980658:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764177.980663:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764177.980665:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764177.980667:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764177.980670:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764177.980674:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764177.980676:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787764177.980680:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764177.980680:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:1.0:1787764177.980684:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764177.980687:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933840.
00000100:00000040:2.0:1787764177.980688:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000020:00000040:1.0:1787764177.980691:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764177.980694:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:2.0:1787764177.980694:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
02000000:00000001:1.0:1787764177.980696:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764177.980699:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000200:2.0:1787764177.980700:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832698752, offset 0
02000000:00000001:1.0:1787764177.980701:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764177.980704:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764177.980709:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8c00.
00000400:00000200:2.0:1787764177.980710:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787764177.980713:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764177.980718:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933840 x1874599832699008/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787764177.980721:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764177.980732:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:1.0:1787764177.980739:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:2.0:1787764177.980741:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1787764177.980742:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764177.980748:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000040:1.0:1787764177.980750:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000083dd6d32] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933840 x1874599832699008/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764177.980754:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764177.980760:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:1.0:1787764177.980764:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764177.980768:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58694:58694:256:4294967295] 192.168.203.110@tcp LPNI seq info [58694:58694:8:4294967295]
00000100:00000040:1.0:1787764177.980769:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787764177.980777:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1787764177.980778:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:1.0:1787764177.980781:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764177.980786:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787764177.980790:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764177.980793:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1787764177.980795:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:1.0:1787764177.980797:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764177.980804:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21892
00000800:00000200:2.0:1787764177.980809:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764177.980815:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000010:2.0:1787764177.980820:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000400:00000200:1.0:1787764177.980822:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764177.980827:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764177.980833:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:2.0:1787764177.980833:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787764177.980840:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1787764177.980842:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000100:00000040:1.0:1787764177.980843:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764177.980847:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764177.980850:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764177.980851:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787764177.980852:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:2.1:1787764177.980855:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:1.0:1787764177.980856:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787764177.980861:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000800:00000200:2.1:1787764177.980862:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000020:00000001:1.0:1787764177.980864:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000800:00000100:2.0:1787764177.980869:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764177.980871:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000100:00000040:1.0:1787764177.980878:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764177.980880:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:1.0:1787764177.980886:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764177.980891:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764177.980895:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787764177.980896:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000020:00000040:1.0:1787764177.980899:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:1.0:1787764177.980904:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764177.980906:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764177.980906:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
02000000:00000001:1.0:1787764177.980910:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:2.0:1787764177.980912:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
02000000:00000001:1.0:1787764177.980913:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764177.980918:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764177.980920:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764177.980921:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000008:00100000:1.0:1787764177.980922:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1787764177.980926:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764177.980927:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764177.980931:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930780 x1874599832698752/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:1.0:1787764177.980932:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932940.
00000020:00000040:1.0:1787764177.980935:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787764177.980940:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764177.980942:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764177.980947:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764177.980950:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764177.980951:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764177.980952:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000010:1.0:1787764177.980956:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca400.
00000100:00000040:2.0:1787764177.980956:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930780 x1874599832698752/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764202 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764177.980961:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764177.980968:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764177.980971:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932940 x1874599832699136/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764177.980971:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764177.980979:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764177.980979:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764177.980983:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764177.980987:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:1.0:1787764177.980990:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000af90eaa] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932940 x1874599832699136/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.980991:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316566259
00000100:00000010:2.0:1787764177.981013:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7b40.
00000100:00000001:1.0:1787764177.981014:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764177.981020:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21902)
00000100:00000001:2.0:1787764177.981022:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787764177.981029:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316566259
00000100:00000010:2.0:1787764177.981032:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7ea0.
00000100:00000001:2.0:1787764177.981035:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787764177.981038:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764177.981040:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764177.981042:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764177.981047:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764177/real 1787764177]  req@ffff9751d7930780 x1874599832698752/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764177.981054:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764177.981055:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764177.981057:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764177.981059:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764177.981060:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764177.981062:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764177.981064:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930780 x1874599832698752/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764177.981070:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930780 x1874599832698752/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981075:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764177.981076:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.981078:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764177.981079:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764177.981082:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930780 x1874599832698752/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981086:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764177.981090:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930780 x1874599832698752/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981095:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764177.981097:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.981098:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764177.981098:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.981100:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764177.981101:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764177.981103:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764177.981106:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764177.981107:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764177.981110:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930780 x1874599832698752/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764177.981115:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832698752:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764177.981120:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.981122:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764177.981124:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930780 x1874599832698752/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981128:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764177.981130:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764177.981132:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764177.981150:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764177.981152:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764177.981153:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764177.981154:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764177.981156:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8400.
02000000:00000001:2.0:1787764177.981159:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764177.981160:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764177.981162:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930780.
00000100:00000001:2.0:1787764177.981164:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764177.981165:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.981166:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.981171:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764177.981172:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764177.981174:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764177.981175:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764177.981176:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764177.981179:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932580 x1874599832698880/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981184:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764177.981185:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764177.981186:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764177.981188:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764177.981189:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764177.981191:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832698880:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764177.981193:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764177.981195:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764177.981196:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764177.981198:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764177.981200:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764177.981202:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764177.981204:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764177.981206:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764177.981208:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764177.981210:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832698880, portal 10
00000100:00000001:2.0:1787764177.981212:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764177.981213:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497730944 : -115097211820672 : ffff9751d7932580)
00000100:00000040:2.0:1787764177.981218:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932580 x1874599832698880/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981222:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764177.981224:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764177.981225:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764177.981227:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832698880, offset 0
00000400:00000200:2.0:1787764177.981229:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764177.981232:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764177.981235:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764177.981237:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764177.981239:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764177.981241:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764177.981242:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764177.981245:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58695:58695:256:4294967295] 192.168.203.110@tcp LPNI seq info [58695:58695:8:4294967295]
00000400:00000200:2.0:1787764177.981248:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764177.981252:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764177.981255:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764177.981257:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764177.981260:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764177.981262:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787764177.981266:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764177.981268:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:2.0:1787764177.981270:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764177.981272:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764177.981274:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764177.981277:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764177.981279:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764177.981280:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764177.981282:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764177.981284:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932580 x1874599832698880/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981289:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764177.981291:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932580 x1874599832698880/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764202 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981296:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764177.981297:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764177.981299:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764177.981302:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764177.981304:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764177.981305:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316565024
00000100:00000001:2.0:1787764177.981307:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316566259
00000100:00000001:2.0:1787764177.981308:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316565024
00000100:00000001:2.0:1787764177.981310:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316566259
00000100:00000001:2.0:1787764177.981312:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764177.981313:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764177.981314:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764177.981316:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764177/real 1787764177]  req@ffff9751d7932580 x1874599832698880/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764177.981321:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764177.981322:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764177.981323:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764177.981324:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764177.981325:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764177.981327:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764177.981329:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932580 x1874599832698880/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764177.981334:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932580 x1874599832698880/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981338:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764177.981339:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.981340:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764177.981341:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764177.981343:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932580 x1874599832698880/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981347:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764177.981349:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932580 x1874599832698880/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981353:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764177.981355:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.981356:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764177.981357:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.981358:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764177.981367:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764177.981369:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764177.981371:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764177.981372:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764177.981374:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932580 x1874599832698880/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764177.981379:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832698880:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764177.981382:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764177.981384:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933840 x1874599832699008/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981393:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764177.981394:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764177.981395:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764177.981397:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764177.981398:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764177.981400:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832699008:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764177.981401:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764177.981403:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764177.981403:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764177.981404:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764177.981407:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764177.981408:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764177.981410:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764177.981412:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764177.981414:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764177.981415:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832699008, portal 10
00000100:00000001:2.0:1787764177.981417:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764177.981418:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497735744 : -115097211815872 : ffff9751d7933840)
00000100:00000040:2.0:1787764177.981422:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933840 x1874599832699008/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981426:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764177.981428:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764177.981429:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764177.981430:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832699008, offset 0
00000400:00000200:2.0:1787764177.981433:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764177.981435:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764177.981437:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764177.981438:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764177.981440:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764177.981441:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764177.981443:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764177.981445:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58696:58696:256:4294967295] 192.168.203.110@tcp LPNI seq info [58696:58696:8:4294967295]
00000400:00000200:2.0:1787764177.981448:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764177.981452:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764177.981455:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764177.981457:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764177.981459:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764177.981461:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787764177.981464:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764177.981466:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:2.0:1787764177.981469:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764177.981471:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764177.981472:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764177.981475:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764177.981476:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764177.981477:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764177.981479:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764177.981481:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933840 x1874599832699008/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981486:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764177.981488:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933840 x1874599832699008/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764202 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981492:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764177.981493:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764177.981495:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764177.981497:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764177.981498:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764177.981500:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316565012
00000100:00000001:2.0:1787764177.981501:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316566259
00000100:00000001:2.0:1787764177.981503:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316565012
00000100:00000001:2.0:1787764177.981504:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316566259
00000100:00000001:2.0:1787764177.981506:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764177.981508:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764177.981509:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764177.981511:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764177/real 1787764177]  req@ffff9751d7933840 x1874599832699008/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764177.981516:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764177.981517:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764177.981518:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764177.981519:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764177.981520:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764177.981521:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764177.981523:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933840 x1874599832699008/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764177.981529:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933840 x1874599832699008/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981533:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764177.981534:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.981535:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764177.981535:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764177.981538:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933840 x1874599832699008/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981541:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764177.981544:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933840 x1874599832699008/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981548:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764177.981549:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.981550:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764177.981551:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.981552:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764177.981553:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764177.981554:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764177.981556:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764177.981558:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764177.981560:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933840 x1874599832699008/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764177.981565:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832699008:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764177.981567:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764177.981569:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932940 x1874599832699136/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981573:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764177.981574:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764177.981575:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764177.981576:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764177.981577:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764177.981579:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832699136:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764177.981581:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764177.981582:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764177.981583:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764177.981584:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764177.981586:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764177.981587:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764177.981589:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764177.981590:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764177.981592:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764177.981594:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832699136, portal 4
00000100:00000001:2.0:1787764177.981595:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764177.981596:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497731904 : -115097211819712 : ffff9751d7932940)
00000100:00000040:2.0:1787764177.981600:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932940 x1874599832699136/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981604:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764177.981606:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764177.981607:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764177.981608:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832699136, offset 0
00000400:00000200:2.0:1787764177.981611:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764177.981613:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764177.981615:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764177.981616:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764177.981618:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764177.981620:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764177.981621:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764177.981623:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58697:58697:256:4294967295] 192.168.203.110@tcp LPNI seq info [58697:58697:8:4294967295]
00000400:00000200:2.0:1787764177.981626:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764177.981630:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764177.981633:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764177.981634:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764177.981636:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764177.981638:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:2.1:1787764177.981640:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764177.981643:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:2.0:1787764177.981645:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764177.981646:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764177.981648:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764177.981650:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764177.981652:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764177.981653:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764177.981654:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764177.981656:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932940 x1874599832699136/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981661:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764177.981663:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932940 x1874599832699136/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764202 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981667:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764177.981668:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764177.981670:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764177.981672:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764177.981674:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764177.981676:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316565029
00000100:00000001:2.0:1787764177.981677:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316566259
00000100:00000001:2.0:1787764177.981679:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316565029
00000100:00000001:2.0:1787764177.981680:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316566259
00000100:00000001:2.0:1787764177.981681:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764177.981682:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764177.981683:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764177.981686:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764177/real 1787764177]  req@ffff9751d7932940 x1874599832699136/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764177.981690:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764177.981691:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764177.981692:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764177.981693:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764177.981694:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764177.981695:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764177.981697:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932940 x1874599832699136/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764177.981703:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932940 x1874599832699136/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981706:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764177.981707:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.981708:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764177.981709:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764177.981711:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932940 x1874599832699136/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981715:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764177.981717:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932940 x1874599832699136/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981722:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764177.981724:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.981725:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764177.981725:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.981726:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764177.981727:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764177.981729:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764177.981731:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764177.981732:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764177.981734:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932940 x1874599832699136/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764177.981739:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832699136:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764177.981741:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.981743:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764177.981746:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932580 x1874599832698880/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981752:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764177.981754:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764177.981756:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764177.981758:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764177.981759:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764177.981760:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764177.981762:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764177.981764:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8800.
02000000:00000001:2.0:1787764177.981767:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764177.981768:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764177.981770:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932580.
00000100:00000001:2.0:1787764177.981773:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764177.981774:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.981776:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764177.981780:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933840 x1874599832699008/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981787:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764177.981789:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764177.981790:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764177.981793:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764177.981794:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764177.981796:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764177.981798:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764177.981799:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8c00.
02000000:00000001:2.0:1787764177.981802:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764177.981803:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764177.981804:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933840.
00000100:00000001:2.0:1787764177.981808:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764177.981809:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.981810:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764177.981813:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932940 x1874599832699136/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764202 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764177.981817:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764177.981818:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764177.981819:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764177.981820:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764177.981821:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764177.981822:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764177.981823:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764177.981824:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca400.
02000000:00000001:2.0:1787764177.981825:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764177.981826:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764177.981826:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932940.
00000100:00000001:2.0:1787764177.981828:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764177.981828:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.981829:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764177.981832:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764177.981833:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764177.981834:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764177.981836:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764178.620216:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764178.620217:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764178.620223:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764178.620224:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764178.620245:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764178.620247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764178.620249:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764178.620250:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764178.620258:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764178.620258:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764178.620262:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764178.620263:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764178.620267:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764178.620269:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764178.620271:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764178.620306:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764178.620311:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764178.620314:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:0.0:1787764179.004277:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787764179.004285:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764179.004288:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764179.004294:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787764179.004298:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764179.004302:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764179.004305:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764179.004309:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764179.004312:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764179.260216:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316566259, at 4316566272
00000100:00000010:3.0:1787764179.260231:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7ea0.
00000100:00000040:2.0:1787764179.260269:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316566259, at 4316566272
00000100:00000010:2.0:1787764179.260278:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7b40.
00000100:00000001:2.0:1787764179.644271:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764179.644274:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764179.644278:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764179.644279:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764179.644281:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764179.644284:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764179.644284:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764179.644286:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764179.644289:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764179.644290:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764179.644291:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764179.644292:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764179.644295:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764179.644298:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764179.644301:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764179.644336:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764179.644340:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764179.644343:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764180.028275:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764180.028281:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764180.028282:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764180.028287:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764180.028290:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764180.028293:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764180.028295:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764180.028297:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764180.028299:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764180.668212:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764180.668217:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764180.668220:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764180.668222:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764180.668225:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764180.668226:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764180.668228:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764180.668229:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764180.668231:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764180.668235:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764180.668239:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764180.668243:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764180.668245:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764180.668248:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764180.668251:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764180.668261:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764180.668266:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764180.668269:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764181.052255:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764181.052261:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764181.052263:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764181.052268:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764181.052272:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764181.052275:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764181.052277:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764181.052280:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764181.052283:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764181.692231:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764181.692233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764181.692238:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764181.692238:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764181.692242:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764181.692244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764181.692246:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764181.692247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764181.692252:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764181.692254:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764181.692256:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764181.692257:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764181.692260:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764181.692270:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764181.692274:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764181.692294:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764181.692298:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764181.692301:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764182.076223:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764182.076236:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764182.076238:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764182.076245:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764182.076248:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764182.076250:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764182.076253:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764182.076254:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764182.076256:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764182.716202:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764182.716202:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764182.716208:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764182.716210:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764182.716214:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764182.716216:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764182.716219:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764182.716219:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764182.716226:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764182.716227:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764182.716230:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764182.716232:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764182.716234:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764182.716235:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764182.716237:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764182.716239:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764182.716241:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764182.716242:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764182.716246:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764182.716248:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764182.716250:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764182.716282:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764182.716285:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764182.716288:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764183.100214:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764183.100225:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764183.100228:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764183.100231:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1787764183.100236:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764183.100243:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:1.0:1787764183.100247:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787764183.100251:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764183.100252:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787764183.100257:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:1.0:1787764183.100259:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764183.100262:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764183.100266:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787764183.100269:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764183.100277:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21897
00000400:00000200:1.0:1787764183.100286:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764183.100295:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764183.100299:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764183.100303:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764183.100306:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764183.100308:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764183.100310:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764183.100314:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764183.100316:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787764183.100318:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764183.100321:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787764183.100323:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787764183.100325:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787764183.100327:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764183.100329:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764183.100331:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764183.100332:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764183.100335:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787764183.100337:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764183.100339:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764183.100341:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764183.100345:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764183.100347:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764183.100349:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764183.100351:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764183.100359:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933480.
00000020:00000040:1.0:1787764183.100362:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787764183.100366:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764183.100367:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764183.100370:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764183.100372:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764183.100375:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764183.100384:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca400.
00000100:00000001:1.0:1787764183.100389:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764183.100397:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933480 x1874599832699264/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764183.100406:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764183.100409:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764183.100413:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3c2c33c] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933480 x1874599832699264/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764183.100431:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764183.100434:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764183.100436:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787764183.100438:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764183.100439:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764183.100441:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764183.100443:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21897
00000400:00000200:1.0:1787764183.100447:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764183.100450:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764183.100452:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764183.100455:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764183.100457:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764183.100460:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764183.100461:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764183.100464:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1787764183.100464:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:1.0:1787764183.100466:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787764183.100470:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1787764183.100472:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000040:1.0:1787764183.100473:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787764183.100478:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787764183.100479:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1787764183.100481:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787764183.100487:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764183.100492:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764183.100494:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933480 x1874599832699264/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787764183.100499:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764183.100506:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764183.100512:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787764183.100518:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764183.100521:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764183.100524:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787764183.100524:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764183.100528:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764183.100531:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764183.100534:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764183.100536:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764183.100538:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764183.100542:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1787764183.100543:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:1.0:1787764183.100550:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00100000:2.0:1787764183.100551:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832699264:192.168.203.110@tcp:250:kworker.0
00000008:00000001:1.0:1787764183.100557:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764183.100559:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1787764183.100561:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764183.100566:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:1.0:1787764183.100567:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930000.
02000000:00000001:2.0:1787764183.100569:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1787764183.100571:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764183.100574:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:1.0:1787764183.100576:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764183.100579:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764183.100580:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000010:2.0:1787764183.100581:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:1.0:1787764183.100582:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764183.100587:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764183.100588:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764183.100596:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8c00.
00000400:00000200:2.0:1787764183.100597:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764183.100604:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764183.100605:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000100:00100000:1.0:1787764183.100613:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930000 x1874599832699392/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:2.0:1787764183.100614:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764183.100623:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832699264, portal 25
00000100:00000001:1.0:1787764183.100624:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764183.100629:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764183.100630:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764183.100633:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497734784 : -115097211816832 : ffff9751d7933480)
00000100:00000040:1.0:1787764183.100637:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6d90274] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930000 x1874599832699392/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787764183.100643:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933480 x1874599832699264/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764208 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764183.100678:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764183.100683:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764183.100691:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787764183.100693:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787764183.100693:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:1.0:1787764183.100698:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764183.100702:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:2.0:1787764183.100703:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00080000:1.0:1787764183.100707:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21897
00000400:00000200:1.0:1787764183.100712:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764183.100712:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000400:00000200:1.0:1787764183.100720:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000200:2.0:1787764183.100721:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832699264, offset 0
00000100:00080000:1.0:1787764183.100722:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764183.100729:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764183.100731:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764183.100735:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764183.100738:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787764183.100739:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764183.100745:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764183.100746:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787764183.100747:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764183.100749:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787764183.100751:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787764183.100754:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:2.0:1787764183.100755:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:1.0:1787764183.100758:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764183.100762:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764183.100764:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764183.100766:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764183.100768:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:2.0:1787764183.100770:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00080000:1.0:1787764183.100774:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764183.100778:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764183.100781:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1787764183.100781:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:1.0:1787764183.100786:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764183.100789:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764183.100792:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:1.0:1787764183.100793:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764183.100797:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787764183.100801:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764183.100802:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1787764183.100804:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764183.100810:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930b40.
00000400:00000200:2.0:1787764183.100812:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000020:00000040:1.0:1787764183.100813:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764183.100818:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764183.100820:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764183.100823:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:2.0:1787764183.100824:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58698:58698:256:4294967295] 192.168.203.110@tcp LPNI seq info [58698:58698:8:4294967295]
02000000:00000001:1.0:1787764183.100825:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764183.100828:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764183.100835:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8800.
00000100:00000001:1.0:1787764183.100841:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764183.100841:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00100000:1.0:1787764183.100849:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930b40 x1874599832699520/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764183.100855:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764183.100857:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764183.100859:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022d7f91a] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930b40 x1874599832699520/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764183.100874:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764183.100875:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:1.0:1787764183.100877:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787764183.100886:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787764183.100889:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764183.100895:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000800:00000200:2.0:1787764183.100895:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787764183.100898:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764183.100901:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764183.100904:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000800:00000010:2.0:1787764183.100907:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000100:00080000:1.0:1787764183.100908:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21897
00000400:00000200:1.0:1787764183.100914:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764183.100918:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764183.100921:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000200:2.0:1787764183.100923:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:1.0:1787764183.100927:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764183.100930:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1787764183.100931:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000100:00000040:1.0:1787764183.100932:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764183.100935:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764183.100941:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764183.100943:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787764183.100945:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764183.100948:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000800:00000200:2.1:1787764183.100949:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:1.0:1787764183.100951:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787764183.100956:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000800:00000200:2.1:1787764183.100958:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000100:00000040:1.0:1787764183.100960:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764183.100978:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764183.100980:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764183.100986:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764183.100988:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:1.0:1787764183.100994:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000800:00000100:2.0:1787764183.100994:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787764183.101000:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764183.101002:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000010:2.0:1787764183.101002:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
02000000:00000001:1.0:1787764183.101006:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764183.101009:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764183.101010:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764183.101013:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000400:00000200:2.0:1787764183.101013:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000008:00000001:1.0:1787764183.101020:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764183.101022:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764183.101027:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000010:1.0:1787764183.101028:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7931680.
00000020:00000040:1.0:1787764183.101034:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787764183.101037:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:2.0:1787764183.101039:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
02000000:00000001:1.0:1787764183.101040:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764183.101045:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764183.101047:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:2.0:1787764183.101047:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
02000000:00000001:1.0:1787764183.101050:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764183.101056:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8400.
00000100:00000001:2.0:1787764183.101056:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:1.0:1787764183.101061:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764183.101067:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933480 x1874599832699264/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764208 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764183.101074:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7931680 x1874599832699648/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764183.101094:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787764183.101096:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764183.101101:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764183.101104:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933480 x1874599832699264/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764208 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787764183.101111:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d4255d6] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7931680 x1874599832699648/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764183.101157:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764183.101161:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21907)
00000100:00000001:2.0:1787764183.101173:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764183.101177:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764183.101185:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764183.101191:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764183.101194:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316571380
00000100:00000010:2.0:1787764183.101200:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7b40.
00000100:00000001:2.0:1787764183.101206:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787764183.101212:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316571380
00000100:00000010:2.0:1787764183.101214:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7c60.
00000100:00000001:2.0:1787764183.101217:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787764183.101219:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764183.101221:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764183.101223:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764183.101228:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764183/real 1787764183]  req@ffff9751d7933480 x1874599832699264/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764183.101235:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764183.101237:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764183.101239:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764183.101241:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764183.101242:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764183.101244:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764183.101247:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933480 x1874599832699264/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764183.101253:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933480 x1874599832699264/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101258:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764183.101259:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.101261:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764183.101263:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764183.101265:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933480 x1874599832699264/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101271:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764183.101275:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933480 x1874599832699264/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101280:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764183.101282:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.101283:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764183.101284:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.101286:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764183.101288:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764183.101290:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764183.101293:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764183.101294:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764183.101297:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933480 x1874599832699264/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764183.101303:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832699264:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764183.101306:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.101308:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764183.101310:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933480 x1874599832699264/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101316:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764183.101318:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764183.101320:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764183.101322:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764183.101323:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764183.101324:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764183.101326:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764183.101328:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca400.
02000000:00000001:2.0:1787764183.101332:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764183.101334:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764183.101335:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933480.
00000100:00000001:2.0:1787764183.101339:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764183.101340:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.101342:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.101347:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764183.101349:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764183.101350:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764183.101352:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764183.101353:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764183.101356:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930000 x1874599832699392/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101361:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764183.101362:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764183.101364:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764183.101366:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764183.101367:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764183.101369:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832699392:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764183.101372:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764183.101373:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764183.101375:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764183.101376:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764183.101378:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764183.101380:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764183.101382:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764183.101385:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764183.101387:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764183.101389:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832699392, portal 10
00000100:00000001:2.0:1787764183.101391:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764183.101392:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497721344 : -115097211830272 : ffff9751d7930000)
00000100:00000040:2.0:1787764183.101397:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930000 x1874599832699392/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764208 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101402:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764183.101404:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764183.101406:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764183.101408:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832699392, offset 0
00000400:00000200:2.0:1787764183.101411:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764183.101414:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764183.101417:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764183.101420:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764183.101422:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764183.101424:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764183.101426:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764183.101428:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58699:58699:256:4294967295] 192.168.203.110@tcp LPNI seq info [58699:58699:8:4294967295]
00000400:00000200:2.0:1787764183.101431:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764183.101437:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764183.101440:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764183.101443:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764183.101446:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764183.101448:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787764183.101452:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764183.101454:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:2.0:1787764183.101457:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764183.101458:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764183.101461:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764183.101463:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764183.101466:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764183.101467:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764183.101469:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764183.101472:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930000 x1874599832699392/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764208 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101478:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764183.101480:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930000 x1874599832699392/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764208 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101485:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764183.101486:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764183.101489:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764183.101492:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764183.101494:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764183.101496:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316570144
00000100:00000001:2.0:1787764183.101498:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316571380
00000100:00000001:2.0:1787764183.101500:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316570144
00000100:00000001:2.0:1787764183.101501:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316571380
00000100:00000001:2.0:1787764183.101503:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764183.101504:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764183.101506:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764183.101508:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764183/real 1787764183]  req@ffff9751d7930000 x1874599832699392/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764183.101514:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764183.101515:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764183.101516:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764183.101518:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764183.101519:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764183.101520:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764183.101522:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930000 x1874599832699392/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764183.101528:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930000 x1874599832699392/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101533:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764183.101534:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.101535:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764183.101536:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764183.101539:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930000 x1874599832699392/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101543:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764183.101546:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930000 x1874599832699392/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101552:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764183.101554:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.101555:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764183.101556:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.101557:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764183.101567:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764183.101569:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764183.101571:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764183.101573:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764183.101576:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930000 x1874599832699392/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764183.101582:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832699392:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764183.101584:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764183.101586:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930b40 x1874599832699520/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101591:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764183.101592:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764183.101593:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764183.101595:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764183.101596:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764183.101598:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832699520:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764183.101600:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764183.101601:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764183.101602:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764183.101604:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764183.101606:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764183.101607:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764183.101610:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764183.101612:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764183.101614:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764183.101616:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832699520, portal 10
00000100:00000001:2.0:1787764183.101617:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764183.101618:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497724224 : -115097211827392 : ffff9751d7930b40)
00000100:00000040:2.0:1787764183.101622:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930b40 x1874599832699520/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764208 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101627:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764183.101629:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764183.101630:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764183.101632:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832699520, offset 0
00000400:00000200:2.0:1787764183.101635:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764183.101637:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764183.101639:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764183.101641:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764183.101643:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764183.101645:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764183.101647:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764183.101649:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58700:58700:256:4294967295] 192.168.203.110@tcp LPNI seq info [58700:58700:8:4294967295]
00000400:00000200:2.0:1787764183.101653:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764183.101660:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764183.101666:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764183.101669:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764183.101674:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764183.101677:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787764183.101682:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764183.101685:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:2.0:1787764183.101690:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764183.101692:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764183.101695:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764183.101699:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764183.101703:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764183.101706:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764183.101709:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764183.101713:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930b40 x1874599832699520/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764208 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101722:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764183.101726:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930b40 x1874599832699520/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764208 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101731:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764183.101732:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764183.101735:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764183.101737:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764183.101740:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764183.101741:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316570132
00000100:00000001:2.0:1787764183.101743:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316571380
00000100:00000001:2.0:1787764183.101745:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316570132
00000100:00000001:2.0:1787764183.101746:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316571380
00000100:00000001:2.0:1787764183.101748:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764183.101749:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764183.101751:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764183.101753:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764183/real 1787764183]  req@ffff9751d7930b40 x1874599832699520/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764183.101759:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764183.101760:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764183.101762:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764183.101763:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764183.101764:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764183.101766:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764183.101768:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930b40 x1874599832699520/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764183.101774:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930b40 x1874599832699520/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101779:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764183.101780:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.101782:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764183.101783:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764183.101785:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930b40 x1874599832699520/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101790:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764183.101793:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930b40 x1874599832699520/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101797:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764183.101799:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.101800:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764183.101801:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.101802:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764183.101804:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764183.101805:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764183.101807:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764183.101809:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764183.101811:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930b40 x1874599832699520/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764183.101817:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832699520:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764183.101820:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764183.101822:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7931680 x1874599832699648/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101827:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764183.101828:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764183.101830:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764183.101832:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764183.101833:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764183.101835:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832699648:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764183.101838:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764183.101839:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764183.101840:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764183.101842:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764183.101844:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764183.101846:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764183.101848:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764183.101850:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764183.101853:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764183.101855:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832699648, portal 4
00000100:00000001:2.0:1787764183.101856:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764183.101857:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497727104 : -115097211824512 : ffff9751d7931680)
00000100:00000040:2.0:1787764183.101861:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7931680 x1874599832699648/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764208 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101866:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764183.101868:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764183.101869:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764183.101871:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832699648, offset 0
00000400:00000200:2.0:1787764183.101874:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764183.101876:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764183.101879:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764183.101881:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764183.101883:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764183.101884:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764183.101886:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764183.101888:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58701:58701:256:4294967295] 192.168.203.110@tcp LPNI seq info [58701:58701:8:4294967295]
00000400:00000200:2.0:1787764183.101892:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764183.101896:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764183.101900:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764183.101903:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764183.101905:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764183.101907:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:2.1:1787764183.101911:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764183.101913:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:2.0:1787764183.101916:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764183.101917:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764183.101919:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764183.101922:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764183.101924:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764183.101925:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764183.101927:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764183.101930:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7931680 x1874599832699648/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764208 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101935:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764183.101937:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7931680 x1874599832699648/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764208 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.101942:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764183.101943:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764183.101945:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764183.101946:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764183.101948:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764183.101950:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316570149
00000100:00000001:2.0:1787764183.101951:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316571380
00000100:00000001:2.0:1787764183.101953:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316570149
00000100:00000001:2.0:1787764183.101955:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316571380
00000100:00000001:2.0:1787764183.101956:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764183.101958:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764183.101959:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764183.101974:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764183/real 1787764183]  req@ffff9751d7931680 x1874599832699648/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764183.101982:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764183.101983:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764183.101985:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764183.101986:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764183.101988:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764183.101989:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764183.101992:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7931680 x1874599832699648/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764183.101998:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7931680 x1874599832699648/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.102003:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764183.102004:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.102005:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764183.102006:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764183.102008:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7931680 x1874599832699648/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.102013:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764183.102016:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7931680 x1874599832699648/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.102021:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764183.102023:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.102024:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764183.102025:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.102026:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764183.102027:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764183.102029:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764183.102031:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764183.102032:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764183.102036:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7931680 x1874599832699648/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764183.102041:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832699648:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764183.102044:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.102045:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764183.102048:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930000 x1874599832699392/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.102052:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764183.102053:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764183.102055:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764183.102057:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764183.102058:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764183.102059:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764183.102061:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764183.102062:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8c00.
02000000:00000001:2.0:1787764183.102064:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764183.102065:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764183.102067:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930000.
00000100:00000001:2.0:1787764183.102068:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764183.102070:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.102071:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764183.102073:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930b40 x1874599832699520/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.102078:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764183.102079:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764183.102080:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764183.102081:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764183.102082:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764183.102083:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764183.102084:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764183.102085:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8800.
02000000:00000001:2.0:1787764183.102086:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764183.102087:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764183.102088:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930b40.
00000100:00000001:2.0:1787764183.102090:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764183.102091:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.102092:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764183.102094:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7931680 x1874599832699648/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764208 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764183.102099:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764183.102100:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764183.102101:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764183.102102:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764183.102103:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764183.102104:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764183.102105:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764183.102106:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8400.
02000000:00000001:2.0:1787764183.102107:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764183.102108:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764183.102109:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7931680.
00000100:00000001:2.0:1787764183.102110:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764183.102111:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.102113:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.102117:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764183.102118:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764183.102120:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764183.102122:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764183.740235:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764183.740236:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764183.740241:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764183.740243:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764183.740247:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764183.740247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764183.740253:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764183.740256:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764183.740258:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764183.740261:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764183.740263:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764183.740264:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764183.740270:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764183.740274:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764183.740278:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764183.740322:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764183.740326:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764183.740328:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764184.124242:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764184.124251:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764184.124253:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764184.124258:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764184.124263:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764184.124266:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764184.124284:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764184.124286:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764184.124289:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764184.380215:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316571380, at 4316571392
00000100:00000010:3.0:1787764184.380228:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7c60.
00000100:00000040:2.0:1787764184.380247:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316571380, at 4316571392
00000100:00000010:2.0:1787764184.380257:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7b40.
00010000:00000001:3.0:1787764184.764216:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764184.764219:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764184.764222:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764184.764226:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764184.764228:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764184.764230:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764184.764231:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764184.764237:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764184.764239:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764184.764240:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764184.764246:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764184.764249:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764184.764253:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764184.764256:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764184.764259:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764184.764285:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764184.764290:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764184.764294:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764185.148246:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764185.148254:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764185.148257:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764185.148263:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764185.148268:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764185.148271:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764185.148274:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764185.148278:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764185.148281:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764185.788252:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764185.788256:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764185.788259:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764185.788265:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764185.788267:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00100000:2.0:1787764185.788270:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764185.788271:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764185.788276:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764185.788278:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764185.788288:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764185.788289:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764185.788297:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764185.788298:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764185.788306:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764185.788307:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764185.788323:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764185.788324:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764185.788353:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764185.788358:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764185.788362:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764185.788363:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764185.788418:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764185.788423:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764185.788427:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764186.172216:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764186.172222:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764186.172224:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764186.172230:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764186.172233:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764186.172237:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764186.172239:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764186.172241:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764186.172244:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764186.812272:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764186.812279:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764186.812286:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764186.812286:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764186.812290:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764186.812295:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764186.812299:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764186.812300:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764186.812304:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764186.812309:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764186.812313:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764186.812317:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764186.812319:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764186.812321:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764186.812324:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764186.812353:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764186.812361:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764186.812366:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764187.196285:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764187.196294:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764187.196298:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764187.196304:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764187.196309:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764187.196314:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764187.196316:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764187.196320:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764187.196323:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764187.836272:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764187.836272:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764187.836282:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764187.836283:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764187.836291:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764187.836294:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764187.836299:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764187.836300:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764187.836306:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764187.836307:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764187.836315:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764187.836315:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764187.836326:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764187.836326:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764187.836334:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764187.836334:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764187.836349:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764187.836349:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764187.836362:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764187.836371:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764187.836373:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764187.836428:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764187.836433:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764187.836437:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787764188.220250:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764188.220265:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787764188.220269:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764188.220284:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764188.220290:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764188.220301:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21902
00000100:00000001:2.0:1787764188.220312:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000400:00000200:1.0:1787764188.220313:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764188.220322:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000400:00000200:1.0:1787764188.220329:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787764188.220330:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764188.220342:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00100000:2.0:1787764188.220344:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764188.220352:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1787764188.220353:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764188.220362:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764188.220363:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764188.220370:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764188.220372:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764188.220381:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764188.220381:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.220395:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764188.220397:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764188.220407:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787764188.220412:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764188.220417:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787764188.220420:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787764188.220422:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787764188.220425:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764188.220426:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764188.220430:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764188.220432:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764188.220435:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787764188.220439:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764188.220441:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764188.220443:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764188.220447:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764188.220449:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764188.220451:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764188.220454:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764188.220466:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7931680.
00000020:00000040:1.0:1787764188.220469:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787764188.220473:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764188.220474:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764188.220477:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764188.220479:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764188.220485:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764188.220495:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb800.
00000100:00000001:1.0:1787764188.220503:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764188.220512:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7931680 x1874599832699776/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764188.220521:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764188.220525:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764188.220529:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d4255d6] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7931680 x1874599832699776/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764188.220565:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764188.220568:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764188.220571:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787764188.220572:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764188.220574:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764188.220576:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764188.220578:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21902
00000400:00000200:1.0:1787764188.220580:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764188.220584:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764188.220586:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764188.220589:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764188.220591:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764188.220594:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764188.220596:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764188.220597:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764188.220598:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787764188.220599:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764188.220601:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787764188.220602:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787764188.220604:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:1.0:1787764188.220605:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787764188.220617:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1787764188.220622:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764188.220622:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1787764188.220630:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764188.220632:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764188.220633:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764188.220635:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787764188.220641:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764188.220643:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764188.220645:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7931680 x1874599832699776/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787764188.220647:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764188.220656:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764188.220663:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764188.220668:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764188.220675:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:2.0:1787764188.220682:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00000001:1.0:1787764188.220683:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764188.220689:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764188.220690:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764188.220693:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930b40.
00000020:00000040:1.0:1787764188.220697:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764188.220699:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764188.220699:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764188.220702:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764188.220705:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764188.220706:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764188.220708:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764188.220708:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000010:1.0:1787764188.220715:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca000.
00000100:00000001:1.0:1787764188.220718:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764188.220723:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930b40 x1874599832699904/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:2.0:1787764188.220735:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764188.220736:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764188.220740:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764188.220744:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022d7f91a] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930b40 x1874599832699904/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764188.220762:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764188.220769:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:2.0:1787764188.220770:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832699776:192.168.203.110@tcp:250:kworker.0
00000100:00080000:1.0:1787764188.220773:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787764188.220779:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764188.220784:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764188.220788:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787764188.220788:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:1.0:1787764188.220795:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21902
00000400:00000200:1.0:1787764188.220799:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764188.220802:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
02000000:00000001:2.0:1787764188.220802:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:1.0:1787764188.220813:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
02000000:00000001:2.0:1787764188.220813:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764188.220826:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:1.0:1787764188.220828:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764188.220832:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764188.220833:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764188.220835:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764188.220836:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764188.220838:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
02000000:00000010:2.0:1787764188.220839:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
00000020:00000001:1.0:1787764188.220841:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764188.220847:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
02000000:00000001:2.0:1787764188.220848:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764188.220858:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787764188.220864:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:2.0:1787764188.220870:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:1.0:1787764188.220876:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000010:2.0:1787764188.220878:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000100:00000001:1.0:1787764188.220890:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1787764188.220893:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000040:1.0:1787764188.220895:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764188.220903:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000200:2.0:1787764188.220904:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832699776, portal 25
00000100:00000001:2.0:1787764188.220911:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764188.220915:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497727104 : -115097211824512 : ffff9751d7931680)
00000020:00000040:1.0:1787764188.220917:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787764188.220955:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000040:2.0:1787764188.220957:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7931680 x1874599832699776/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764213 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764188.220966:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764188.220978:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764188.220985:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764188.220987:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:1.0:1787764188.221005:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764188.221011:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764188.221018:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000040:2.0:1787764188.221019:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764188.221025:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000008:00000001:1.0:1787764188.221026:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764188.221041:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764188.221043:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832699776, offset 0
00000100:00000010:1.0:1787764188.221049:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930000.
00000020:00000040:1.0:1787764188.221053:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000400:00000200:2.0:1787764188.221056:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
02000000:00000001:1.0:1787764188.221058:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764188.221063:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764188.221065:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764188.221067:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:2.0:1787764188.221071:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:1.0:1787764188.221073:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764188.221080:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9000.
00000400:00000200:2.0:1787764188.221085:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:1.0:1787764188.221087:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764188.221092:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:1.0:1787764188.221098:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930000 x1874599832700032/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:2.0:1787764188.221102:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764188.221118:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1787764188.221123:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764188.221128:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764188.221129:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764188.221137:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58702:58702:256:4294967295] 192.168.203.110@tcp LPNI seq info [58702:58702:8:4294967295]
00000400:00000200:2.0:1787764188.221144:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:1.0:1787764188.221162:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6d90274] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930000 x1874599832700032/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764188.221172:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764188.221177:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787764188.221187:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:1.0:1787764188.221192:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000800:00000200:2.0:1787764188.221199:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00080000:1.0:1787764188.221203:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:2.0:1787764188.221206:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000100:00080000:1.0:1787764188.221213:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787764188.221220:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:2.0:1787764188.221221:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00080000:1.0:1787764188.221230:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:2.0:1787764188.221230:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000100:00000001:1.0:1787764188.221239:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764188.221245:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21902
00000800:00000200:2.1:1787764188.221246:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764188.221252:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000400:00000200:1.0:1787764188.221253:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764188.221268:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000800:00000100:2.0:1787764188.221270:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00080000:1.0:1787764188.221276:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000010:2.0:1787764188.221277:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000100:00000040:1.0:1787764188.221290:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764188.221290:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:1.0:1787764188.221301:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764188.221310:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764188.221312:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:1.0:1787764188.221321:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764188.221325:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:2.0:1787764188.221325:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000020:00000040:1.0:1787764188.221333:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000400:00000010:2.0:1787764188.221334:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000020:00000001:1.0:1787764188.221338:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:2.0:1787764188.221341:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000020:00000040:1.0:1787764188.221342:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787764188.221348:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000200:2.0:1787764188.221352:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7931680 x1874599832699776/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764213 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1787764188.221355:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:1.0:1787764188.221370:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764188.221380:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764188.221385:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764188.221397:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764188.221399:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:1.0:1787764188.221403:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764188.221403:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787764188.221411:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764188.221417:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:2.0:1787764188.221418:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7931680 x1874599832699776/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764213 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787764188.221428:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764188.221435:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764188.221436:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764188.221451:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764188.221451:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000008:00100000:1.0:1787764188.221464:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1787764188.221468:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764188.221468:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764188.221472:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764188.221477:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933480.
00000400:00000200:2.0:1787764188.221477:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:1.0:1787764188.221483:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:2.0:1787764188.221484:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316576500
02000000:00000001:1.0:1787764188.221489:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764188.221491:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764188.221494:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000010:2.0:1787764188.221495:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7b40.
02000000:00000001:1.0:1787764188.221497:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764188.221504:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764188.221507:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
02000000:00000010:1.0:1787764188.221510:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb400.
00000100:00000001:1.0:1787764188.221516:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764188.221518:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316576500
00000100:00100000:1.0:1787764188.221525:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933480 x1874599832700160/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:2.0:1787764188.221527:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb73c0.
00000100:00000001:2.0:1787764188.221535:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787764188.221545:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764188.221554:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764188.221556:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764188.221560:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764188.221565:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000040:1.0:1787764188.221570:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3c2c33c] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933480 x1874599832700160/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764188.221575:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764188/real 1787764188]  req@ffff9751d7931680 x1874599832699776/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764188.221615:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:2.0:1787764188.221620:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:1.0:1787764188.221624:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21912)
00000400:00000001:2.0:1787764188.221624:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764188.221631:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764188.221635:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764188.221637:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764188.221642:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764188.221648:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7931680 x1874599832699776/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764188.221657:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7931680 x1874599832699776/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.221663:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764188.221664:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.221666:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764188.221668:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764188.221671:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7931680 x1874599832699776/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.221676:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764188.221680:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7931680 x1874599832699776/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.221686:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764188.221688:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.221690:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764188.221691:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.221694:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764188.221696:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764188.221699:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764188.221702:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764188.221704:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764188.221707:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7931680 x1874599832699776/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764188.221714:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832699776:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764188.221717:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.221719:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764188.221722:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7931680 x1874599832699776/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.221727:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764188.221729:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764188.221732:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764188.221734:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764188.221736:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764188.221738:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764188.221740:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764188.221742:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb800.
02000000:00000001:2.0:1787764188.221745:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764188.221747:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764188.221749:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7931680.
00000100:00000001:2.0:1787764188.221752:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764188.221753:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.221755:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.221761:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764188.221762:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764188.221764:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764188.221783:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764188.221785:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764188.221790:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930b40 x1874599832699904/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.221797:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764188.221799:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764188.221801:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764188.221804:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764188.221806:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764188.221808:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832699904:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764188.221812:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764188.221814:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764188.221816:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764188.221818:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764188.221821:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764188.221823:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764188.221827:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764188.221830:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764188.221833:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764188.221836:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832699904, portal 10
00000100:00000001:2.0:1787764188.221838:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764188.221840:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497724224 : -115097211827392 : ffff9751d7930b40)
00000100:00000040:2.0:1787764188.221847:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930b40 x1874599832699904/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764213 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.221853:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764188.221855:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764188.221857:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764188.221860:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832699904, offset 0
00000400:00000200:2.0:1787764188.221863:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764188.221867:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764188.221871:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764188.221874:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764188.221877:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764188.221880:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764188.221883:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764188.221886:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58703:58703:256:4294967295] 192.168.203.110@tcp LPNI seq info [58703:58703:8:4294967295]
00000400:00000200:2.0:1787764188.221890:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764188.221897:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764188.221902:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764188.221904:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764188.221908:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764188.221911:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787764188.221916:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764188.221918:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1787764188.221922:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764188.221924:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764188.221948:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764188.221951:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764188.221955:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764188.221956:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764188.221959:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764188.221962:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930b40 x1874599832699904/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764213 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.221970:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764188.221989:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930b40 x1874599832699904/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764213 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.221995:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764188.221996:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764188.221999:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764188.222003:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764188.222006:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764188.222008:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316575264
00000100:00000001:2.0:1787764188.222010:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316576500
00000100:00000001:2.0:1787764188.222012:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316575264
00000100:00000001:2.0:1787764188.222014:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316576500
00000100:00000001:2.0:1787764188.222016:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764188.222018:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764188.222020:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764188.222023:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764188/real 1787764188]  req@ffff9751d7930b40 x1874599832699904/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764188.222029:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764188.222030:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764188.222031:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764188.222033:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764188.222034:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764188.222036:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764188.222038:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930b40 x1874599832699904/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764188.222045:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930b40 x1874599832699904/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.222050:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764188.222052:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.222053:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764188.222054:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764188.222057:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930b40 x1874599832699904/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.222062:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764188.222065:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930b40 x1874599832699904/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.222070:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764188.222072:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.222073:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764188.222074:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.222075:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764188.222086:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764188.222089:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764188.222092:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764188.222094:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764188.222097:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930b40 x1874599832699904/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764188.222103:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832699904:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764188.222106:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764188.222109:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930000 x1874599832700032/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.222114:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764188.222115:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764188.222117:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764188.222119:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764188.222120:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764188.222122:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832700032:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764188.222125:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764188.222127:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764188.222128:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764188.222130:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764188.222133:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764188.222135:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764188.222137:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764188.222139:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764188.222157:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764188.222160:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832700032, portal 10
00000100:00000001:2.0:1787764188.222162:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764188.222183:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497721344 : -115097211830272 : ffff9751d7930000)
00000100:00000040:2.0:1787764188.222189:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930000 x1874599832700032/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764213 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.222199:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764188.222201:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764188.222203:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764188.222205:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832700032, offset 0
00000400:00000200:2.0:1787764188.222208:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764188.222212:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764188.222215:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764188.222217:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764188.222219:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764188.222221:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764188.222223:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764188.222226:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58704:58704:256:4294967295] 192.168.203.110@tcp LPNI seq info [58704:58704:8:4294967295]
00000400:00000200:2.0:1787764188.222229:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764188.222234:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764188.222237:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764188.222239:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764188.222242:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764188.222245:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787764188.222248:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764188.222250:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1787764188.222253:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764188.222254:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764188.222257:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764188.222259:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764188.222262:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764188.222263:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764188.222265:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764188.222267:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930000 x1874599832700032/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764213 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.222274:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764188.222276:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930000 x1874599832700032/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764213 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.222286:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764188.222287:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764188.222290:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764188.222292:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764188.222295:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764188.222296:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316575253
00000100:00000001:2.0:1787764188.222298:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316576500
00000100:00000001:2.0:1787764188.222300:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316575253
00000100:00000001:2.0:1787764188.222302:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316576500
00000100:00000001:2.0:1787764188.222303:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764188.222305:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764188.222307:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764188.222309:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764188/real 1787764188]  req@ffff9751d7930000 x1874599832700032/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764188.222315:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764188.222315:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764188.222317:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764188.222318:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764188.222320:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764188.222321:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764188.222324:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930000 x1874599832700032/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764188.222330:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930000 x1874599832700032/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.222334:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764188.222335:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.222337:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764188.222338:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764188.222340:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930000 x1874599832700032/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.222346:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764188.222349:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930000 x1874599832700032/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.222354:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764188.222356:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.222357:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764188.222358:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.222359:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764188.222360:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764188.222362:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764188.222364:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764188.222366:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764188.222369:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930000 x1874599832700032/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764188.222374:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832700032:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764188.222377:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764188.222379:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933480 x1874599832700160/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.222384:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764188.222386:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764188.222387:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764188.222389:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764188.222391:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764188.222393:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832700160:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764188.222395:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764188.222397:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764188.222398:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764188.222400:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764188.222402:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764188.222420:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764188.222423:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764188.222425:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764188.222428:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764188.222430:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832700160, portal 4
00000100:00000001:2.0:1787764188.222431:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764188.222433:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497734784 : -115097211816832 : ffff9751d7933480)
00000100:00000040:2.0:1787764188.222438:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933480 x1874599832700160/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764213 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.222443:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764188.222445:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764188.222447:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764188.222448:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832700160, offset 0
00000400:00000200:2.0:1787764188.222451:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764188.222454:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764188.222456:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764188.222458:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764188.222460:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764188.222462:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764188.222464:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764188.222467:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58705:58705:256:4294967295] 192.168.203.110@tcp LPNI seq info [58705:58705:8:4294967295]
00000400:00000200:2.0:1787764188.222488:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764188.222493:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764188.222497:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764188.222500:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764188.222503:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764188.222506:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000200:2.1:1787764188.222509:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764188.222512:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 33 secs later)
00000800:00000100:2.0:1787764188.222515:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764188.222518:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764188.222520:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764188.222523:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764188.222526:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764188.222527:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764188.222529:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764188.222533:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933480 x1874599832700160/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764213 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.222540:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764188.222542:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933480 x1874599832700160/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764213 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.222549:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764188.222550:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764188.222553:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764188.222555:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764188.222558:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764188.222560:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316575270
00000100:00000001:2.0:1787764188.222562:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316576500
00000100:00000001:2.0:1787764188.222580:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316575270
00000100:00000001:2.0:1787764188.222582:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316576500
00000100:00000001:2.0:1787764188.222584:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764188.222585:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764188.222587:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764188.222590:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764188/real 1787764188]  req@ffff9751d7933480 x1874599832700160/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764188.222595:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764188.222596:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764188.222597:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764188.222599:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764188.222600:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764188.222602:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764188.222605:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933480 x1874599832700160/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764188.222611:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933480 x1874599832700160/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.222616:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764188.222617:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.222618:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764188.222619:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764188.222622:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933480 x1874599832700160/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.222627:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764188.222630:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933480 x1874599832700160/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.222635:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764188.222636:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.222638:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764188.222639:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.222640:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764188.222641:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764188.222643:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764188.222646:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764188.222647:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764188.222650:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933480 x1874599832700160/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764188.222657:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832700160:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764188.222659:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.222661:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764188.222664:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930b40 x1874599832699904/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.222669:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764188.222670:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764188.222672:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764188.222674:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764188.222675:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764188.222676:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764188.222677:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764188.222679:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca000.
02000000:00000001:2.0:1787764188.222681:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764188.222682:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764188.222684:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930b40.
00000100:00000001:2.0:1787764188.222686:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764188.222687:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.222688:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764188.222691:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930000 x1874599832700032/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.222696:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764188.222697:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764188.222698:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764188.222699:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764188.222700:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764188.222701:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764188.222702:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764188.222703:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9000.
02000000:00000001:2.0:1787764188.222705:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764188.222706:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764188.222707:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930000.
00000100:00000001:2.0:1787764188.222708:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764188.222709:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.222710:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764188.222713:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933480 x1874599832700160/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764213 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764188.222717:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764188.222719:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764188.222720:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764188.222721:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764188.222723:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764188.222724:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764188.222725:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764188.222726:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb400.
02000000:00000001:2.0:1787764188.222728:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764188.222728:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764188.222729:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933480.
00000100:00000001:2.0:1787764188.222732:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764188.222733:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.222735:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.222737:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764188.222738:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764188.222740:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764188.222741:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764188.860225:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764188.860229:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764188.860230:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764188.860237:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764188.860238:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764188.860242:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764188.860243:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764188.860245:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764188.860249:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764188.860249:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764188.860256:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764188.860261:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.860264:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764188.860268:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764188.860271:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764188.860305:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764188.860314:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764188.860317:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764189.244233:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764189.244239:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764189.244241:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764189.244245:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764189.244249:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764189.244252:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764189.244254:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764189.244256:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764189.244259:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764189.500250:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316576500, at 4316576512
00000100:00000010:3.0:1787764189.500268:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb73c0.
00000100:00000040:2.0:1787764189.500290:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316576500, at 4316576512
00000100:00000010:2.0:1787764189.500299:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7b40.
00000100:00000001:2.0:1787764189.884223:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764189.884225:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764189.884230:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764189.884231:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764189.884235:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764189.884237:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764189.884239:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764189.884240:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764189.884244:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764189.884245:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764189.884248:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764189.884249:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764189.884253:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764189.884256:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764189.884259:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764189.884290:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764189.884294:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764189.884297:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764190.268216:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764190.268223:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764190.268224:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764190.268229:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764190.268232:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764190.268235:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764190.268237:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764190.268239:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764190.268241:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764190.908239:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764190.908241:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764190.908246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764190.908247:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764190.908251:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764190.908252:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764190.908256:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764190.908256:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764190.908261:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764190.908262:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764190.908263:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764190.908266:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764190.908269:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764190.908271:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764190.908273:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764190.908297:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764190.908302:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764190.908305:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764191.292238:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764191.292244:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764191.292246:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764191.292251:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764191.292254:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764191.292257:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764191.292259:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764191.292262:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764191.292265:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764191.932229:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764191.932230:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764191.932236:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764191.932239:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764191.932242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764191.932243:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764191.932244:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764191.932250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764191.932252:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764191.932253:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764191.932259:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764191.932263:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764191.932266:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764191.932270:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764191.932273:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764191.932284:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764191.932288:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764191.932292:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764192.316206:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764192.316213:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764192.316215:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764192.316220:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764192.316223:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764192.316226:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764192.316228:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764192.316230:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764192.316232:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764192.956225:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764192.956225:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764192.956231:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764192.956232:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764192.956235:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764192.956241:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764192.956241:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764192.956245:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764192.956246:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764192.956250:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764192.956254:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764192.956254:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764192.956259:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764192.956260:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764192.956261:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764192.956265:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764192.956270:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764192.956274:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764192.956277:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764192.956280:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764192.956283:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764192.956321:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764192.956324:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764192.956327:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787764193.340255:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764193.340256:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764193.340262:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:1.0:1787764193.340265:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787764193.340265:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764193.340270:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:2.0:1787764193.340271:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:1.0:1787764193.340273:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764193.340276:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764193.340279:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787764193.340280:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.340284:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:1.0:1787764193.340287:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21907
00000100:00000001:2.0:1787764193.340287:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.340290:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764193.340296:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764193.340304:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764193.340306:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764193.340310:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764193.340313:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764193.340318:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764193.340321:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764193.340325:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764193.340326:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787764193.340329:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764193.340332:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787764193.340334:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787764193.340336:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787764193.340338:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764193.340340:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764193.340344:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764193.340346:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764193.340348:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787764193.340351:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764193.340353:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764193.340356:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764193.340360:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764193.340363:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764193.340364:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764193.340367:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764193.340375:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932940.
00000020:00000040:1.0:1787764193.340377:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787764193.340380:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764193.340382:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764193.340384:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764193.340386:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764193.340389:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764193.340396:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8000.
00000100:00000001:1.0:1787764193.340400:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764193.340407:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932940 x1874599832700288/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764193.340415:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764193.340418:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764193.340422:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000af90eaa] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932940 x1874599832700288/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764193.340438:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764193.340440:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764193.340443:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787764193.340444:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764193.340445:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764193.340447:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764193.340449:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21907
00000400:00000200:1.0:1787764193.340451:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764193.340452:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764193.340454:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764193.340457:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764193.340458:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764193.340461:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764193.340463:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764193.340465:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764193.340466:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787764193.340467:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764193.340469:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000001:2.0:1787764193.340470:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:1.0:1787764193.340471:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787764193.340474:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787764193.340474:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:1.0:1787764193.340477:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764193.340478:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1787764193.340482:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764193.340486:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764193.340488:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:2.0:1787764193.340488:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932940 x1874599832700288/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787764193.340494:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787764193.340500:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764193.340505:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764193.340508:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787764193.340514:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:1.0:1787764193.340515:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764193.340518:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764193.340519:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764193.340524:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:2.0:1787764193.340526:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:1.0:1787764193.340528:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787764193.340531:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764193.340531:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764193.340535:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764193.340538:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764193.340539:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933840.
00000020:00000040:1.0:1787764193.340543:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764193.340547:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764193.340548:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764193.340548:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832700288:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787764193.340552:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764193.340554:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787764193.340557:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764193.340558:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764193.340564:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:2.0:1787764193.340565:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1787764193.340567:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764193.340568:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764193.340572:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00100000:1.0:1787764193.340575:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933840 x1874599832700416/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:2.0:1787764193.340578:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764193.340582:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764193.340586:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764193.340589:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764193.340591:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:1.0:1787764193.340595:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000083dd6d32] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933840 x1874599832700416/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1787764193.340595:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000100:00000001:1.0:1787764193.340604:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:2.0:1787764193.340604:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00080000:1.0:1787764193.340609:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764193.340612:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000200:2.0:1787764193.340612:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832700288, portal 25
00000100:00000001:1.0:1787764193.340616:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787764193.340617:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00080000:1.0:1787764193.340619:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764193.340619:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497731904 : -115097211819712 : ffff9751d7932940)
00000100:00000001:1.0:1787764193.340625:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764193.340628:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21907
00000100:00000040:2.0:1787764193.340629:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932940 x1874599832700288/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764218 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787764193.340634:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764193.340639:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787764193.340643:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:1.0:1787764193.340644:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764193.340650:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000040:1.0:1787764193.340651:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764193.340654:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:2.0:1787764193.340654:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000040:1.0:1787764193.340659:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000200:2.0:1787764193.340659:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832700288, offset 0
00000100:00000001:1.0:1787764193.340664:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764193.340665:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764193.340668:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000400:00000200:2.0:1787764193.340669:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000020:00000001:1.0:1787764193.340670:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764193.340673:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787764193.340675:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787764193.340676:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787764193.340680:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764193.340681:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764193.340683:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764193.340686:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764193.340688:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787764193.340689:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000020:00000040:1.0:1787764193.340691:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:2.0:1787764193.340696:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:1.0:1787764193.340697:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764193.340701:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764193.340703:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:2.0:1787764193.340703:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
02000000:00000001:1.0:1787764193.340707:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1787764193.340708:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
02000000:00000001:1.0:1787764193.340710:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764193.340713:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:1.0:1787764193.340714:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764193.340717:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:2.0:1787764193.340718:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58706:58706:256:4294967295] 192.168.203.110@tcp LPNI seq info [58706:58706:8:4294967295]
00000008:00000001:1.0:1787764193.340721:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764193.340725:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764193.340725:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000010:1.0:1787764193.340730:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932580.
00000020:00000040:1.0:1787764193.340734:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000400:00000200:2.0:1787764193.340738:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
02000000:00000001:1.0:1787764193.340739:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764193.340742:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764193.340745:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764193.340748:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:2.0:1787764193.340748:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
02000000:00000001:1.0:1787764193.340750:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1787764193.340754:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
02000000:00000010:1.0:1787764193.340755:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cac00.
00000100:00000001:1.0:1787764193.340759:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:2.0:1787764193.340763:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00100000:1.0:1787764193.340767:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932580 x1874599832700544/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:2.0:1787764193.340768:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787764193.340779:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764193.340784:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:1.0:1787764193.340787:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000100:2.0:1787764193.340789:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787764193.340793:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:2.0:1787764193.340793:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764193.340799:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:1.0:1787764193.340802:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d026924] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932580 x1874599832700544/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764193.340805:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764193.340811:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764193.340814:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764193.340819:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:1.0:1787764193.340821:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764193.340823:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932940 x1874599832700288/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764218 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787764193.340826:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787764193.340833:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787764193.340835:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764193.340838:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787764193.340839:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764193.340842:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764193.340845:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787764193.340845:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00080000:1.0:1787764193.340851:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21907
00000100:00000040:2.0:1787764193.340852:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932940 x1874599832700288/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764218 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787764193.340860:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764193.340866:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787764193.340866:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764193.340869:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:1.0:1787764193.340871:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764193.340875:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764193.340877:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764193.340877:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764193.340882:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764193.340886:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764193.340887:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764193.340889:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316581619
00000020:00000001:1.0:1787764193.340901:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764193.340903:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787764193.340906:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764193.340909:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000010:2.0:1787764193.340910:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7b40.
00000020:00000001:1.0:1787764193.340911:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787764193.340915:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:1.0:1787764193.340918:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764193.340919:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764193.340919:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000040:1.0:1787764193.340924:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764193.340926:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764193.340927:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316581619
00000020:00000040:1.0:1787764193.340929:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000010:2.0:1787764193.340933:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7660.
00000100:00080000:1.0:1787764193.340934:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764193.340938:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764193.340938:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
02000000:00000001:1.0:1787764193.340942:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787764193.340943:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764193.340946:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764193.340947:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764193.340952:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764193.340952:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:1.0:1787764193.340957:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:2.0:1787764193.340960:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764193/real 1787764193]  req@ffff9751d7932940 x1874599832700288/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:1.0:1787764193.340961:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1787764193.340967:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764193.340971:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764193.340978:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930780.
00000400:00000001:2.0:1787764193.340978:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000040:1.0:1787764193.340982:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787764193.340984:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764193.340985:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764193.340986:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000001:2.0:1787764193.340986:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:1.0:1787764193.340989:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:2.0:1787764193.340990:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
02000000:00000001:1.0:1787764193.340993:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764193.340994:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764193.340998:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764193.341000:0:257575:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000010:1.0:1787764193.341002:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9c00.
00000100:00000200:2.0:1787764193.341004:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932940 x1874599832700288/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764193.341008:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764193.341012:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932940 x1874599832700288/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764193.341019:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930780 x1874599832700672/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764193.341020:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764193.341022:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.341026:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764193.341028:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764193.341036:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932940 x1874599832700288/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764193.341040:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764193.341043:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764193.341050:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764193.341051:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3977950] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930780 x1874599832700672/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764193.341061:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764193.341061:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932940 x1874599832700288/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787764193.341066:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21918)
00000100:00000001:2.0:1787764193.341075:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764193.341079:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.341081:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764193.341082:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.341085:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764193.341087:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764193.341091:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764193.341094:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764193.341097:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764193.341100:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932940 x1874599832700288/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764193.341106:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832700288:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764193.341109:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.341110:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764193.341112:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932940 x1874599832700288/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341116:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764193.341118:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764193.341120:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764193.341123:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764193.341124:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764193.341125:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:2.0:1787764193.341126:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764193.341128:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8000.
02000000:00000001:2.0:1787764193.341131:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764193.341152:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764193.341153:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932940.
00000100:00000001:2.0:1787764193.341156:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764193.341157:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.341159:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.341164:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764193.341165:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764193.341167:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764193.341168:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764193.341169:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764193.341172:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933840 x1874599832700416/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341177:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764193.341178:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764193.341179:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764193.341181:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764193.341182:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764193.341185:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832700416:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764193.341188:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764193.341190:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764193.341192:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764193.341194:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764193.341196:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764193.341198:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764193.341201:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764193.341203:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764193.341206:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764193.341209:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832700416, portal 10
00000100:00000001:2.0:1787764193.341210:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764193.341211:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497735744 : -115097211815872 : ffff9751d7933840)
00000100:00000040:2.0:1787764193.341216:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933840 x1874599832700416/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764218 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341220:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764193.341222:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764193.341223:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764193.341225:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832700416, offset 0
00000400:00000200:2.0:1787764193.341227:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764193.341230:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764193.341232:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764193.341235:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764193.341236:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764193.341238:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764193.341240:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764193.341242:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58707:58707:256:4294967295] 192.168.203.110@tcp LPNI seq info [58707:58707:8:4294967295]
00000400:00000200:2.0:1787764193.341245:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764193.341249:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764193.341252:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764193.341255:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764193.341258:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764193.341260:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787764193.341263:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764193.341265:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1787764193.341268:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764193.341269:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764193.341271:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764193.341273:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764193.341275:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764193.341277:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764193.341278:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764193.341280:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933840 x1874599832700416/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764218 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341286:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764193.341288:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933840 x1874599832700416/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764218 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341292:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764193.341293:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764193.341295:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764193.341297:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:2.0:1787764193.341299:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764193.341301:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316580384
00000100:00000001:2.0:1787764193.341302:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316581619
00000100:00000001:2.0:1787764193.341304:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316580384
00000100:00000001:2.0:1787764193.341306:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316581619
00000100:00000001:2.0:1787764193.341307:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764193.341308:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764193.341310:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764193.341312:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764193/real 1787764193]  req@ffff9751d7933840 x1874599832700416/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764193.341322:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764193.341323:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764193.341324:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764193.341325:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764193.341326:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764193.341327:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764193.341329:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933840 x1874599832700416/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764193.341334:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933840 x1874599832700416/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341338:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764193.341339:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.341341:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764193.341342:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764193.341344:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933840 x1874599832700416/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341348:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764193.341351:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933840 x1874599832700416/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341355:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764193.341356:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.341357:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764193.341358:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.341359:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764193.341368:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764193.341370:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764193.341372:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764193.341373:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764193.341377:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933840 x1874599832700416/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764193.341382:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832700416:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764193.341384:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764193.341386:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932580 x1874599832700544/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341390:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764193.341391:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764193.341392:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764193.341393:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764193.341394:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764193.341396:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832700544:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764193.341398:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764193.341399:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764193.341400:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764193.341401:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764193.341403:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764193.341405:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764193.341407:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764193.341408:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764193.341410:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764193.341412:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832700544, portal 10
00000100:00000001:2.0:1787764193.341413:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764193.341414:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497730944 : -115097211820672 : ffff9751d7932580)
00000100:00000040:2.0:1787764193.341418:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932580 x1874599832700544/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764218 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341422:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764193.341424:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764193.341425:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764193.341426:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832700544, offset 0
00000400:00000200:2.0:1787764193.341429:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764193.341432:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764193.341434:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764193.341436:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764193.341438:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764193.341439:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764193.341441:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764193.341443:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58708:58708:256:4294967295] 192.168.203.110@tcp LPNI seq info [58708:58708:8:4294967295]
00000400:00000200:2.0:1787764193.341445:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764193.341449:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764193.341452:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764193.341454:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764193.341456:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764193.341457:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787764193.341460:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764193.341462:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1787764193.341464:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764193.341466:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764193.341467:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764193.341470:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764193.341472:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764193.341473:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764193.341474:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764193.341476:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932580 x1874599832700544/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764218 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341481:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764193.341483:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932580 x1874599832700544/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764218 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341487:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764193.341488:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764193.341490:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764193.341492:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764193.341493:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764193.341495:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316580372
00000100:00000001:2.0:1787764193.341497:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316581619
00000100:00000001:2.0:1787764193.341498:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316580372
00000100:00000001:2.0:1787764193.341500:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316581619
00000100:00000001:2.0:1787764193.341501:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764193.341502:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764193.341503:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764193.341505:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764193/real 1787764193]  req@ffff9751d7932580 x1874599832700544/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764193.341509:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764193.341510:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764193.341511:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764193.341512:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764193.341514:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764193.341515:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764193.341518:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932580 x1874599832700544/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764193.341528:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932580 x1874599832700544/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341534:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764193.341536:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.341538:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764193.341539:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764193.341543:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932580 x1874599832700544/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341551:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764193.341555:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932580 x1874599832700544/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341563:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764193.341566:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.341567:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764193.341568:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.341569:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764193.341571:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764193.341572:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764193.341575:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764193.341576:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764193.341579:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932580 x1874599832700544/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764193.341584:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832700544:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764193.341587:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764193.341589:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930780 x1874599832700672/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341593:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764193.341594:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764193.341595:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764193.341596:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764193.341598:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764193.341599:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832700672:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764193.341602:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764193.341603:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764193.341604:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764193.341605:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764193.341608:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764193.341610:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764193.341612:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764193.341614:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000400:00000010:2.0:1787764193.341616:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
00000100:00000200:2.0:1787764193.341618:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832700672, portal 4
00000100:00000001:2.0:1787764193.341620:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764193.341621:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497723264 : -115097211828352 : ffff9751d7930780)
00000100:00000040:2.0:1787764193.341625:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930780 x1874599832700672/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764218 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341629:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764193.341631:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764193.341632:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000200:2.0:1787764193.341633:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832700672, offset 0
00000400:00000200:2.0:1787764193.341636:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764193.341638:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764193.341641:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764193.341643:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764193.341644:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764193.341646:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764193.341647:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764193.341649:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58709:58709:256:4294967295] 192.168.203.110@tcp LPNI seq info [58709:58709:8:4294967295]
00000400:00000200:2.0:1787764193.341652:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764193.341656:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764193.341659:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764193.341661:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000800:00000200:2.0:1787764193.341663:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.0:1787764193.341665:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:2.1:1787764193.341667:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764193.341669:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:2.0:1787764193.341672:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764193.341673:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764193.341675:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:2.0:1787764193.341677:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:2.0:1787764193.341679:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764193.341680:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764193.341682:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764193.341684:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930780 x1874599832700672/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764218 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341693:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764193.341695:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930780 x1874599832700672/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764218 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341700:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764193.341701:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764193.341702:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764193.341704:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764193.341706:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764193.341707:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316580389
00000100:00000001:2.0:1787764193.341709:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316581619
00000100:00000001:2.0:1787764193.341710:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316580389
00000100:00000001:2.0:1787764193.341712:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316581619
00000100:00000001:2.0:1787764193.341713:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764193.341714:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764193.341716:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:2.0:1787764193.341718:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764193/real 1787764193]  req@ffff9751d7930780 x1874599832700672/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:2.0:1787764193.341722:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:2.0:1787764193.341723:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:2.0:1787764193.341724:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:2.0:1787764193.341725:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:2.0:1787764193.341726:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764193.341728:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:2.0:1787764193.341730:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930780 x1874599832700672/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764193.341735:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930780 x1874599832700672/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341738:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:2.0:1787764193.341740:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.341741:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764193.341742:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764193.341744:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930780 x1874599832700672/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341748:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764193.341750:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930780 x1874599832700672/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341754:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764193.341756:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.341757:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764193.341757:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.341759:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:2.0:1787764193.341759:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:2.0:1787764193.341761:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:2.0:1787764193.341763:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764193.341764:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:2.0:1787764193.341766:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930780 x1874599832700672/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764193.341771:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832700672:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764193.341773:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.341774:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764193.341776:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933840 x1874599832700416/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341780:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764193.341781:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764193.341783:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b9800.
02000000:00000001:2.0:1787764193.341784:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764193.341785:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764193.341786:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764193.341788:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764193.341789:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9400.
02000000:00000001:2.0:1787764193.341791:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764193.341792:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764193.341793:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933840.
00000100:00000001:2.0:1787764193.341795:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764193.341796:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.341797:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764193.341799:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932580 x1874599832700544/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341803:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764193.341804:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764193.341805:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:2.0:1787764193.341807:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764193.341807:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764193.341808:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:2.0:1787764193.341809:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764193.341810:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cac00.
02000000:00000001:2.0:1787764193.341811:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764193.341812:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764193.341813:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932580.
00000100:00000001:2.0:1787764193.341814:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764193.341815:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.341815:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764193.341817:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930780 x1874599832700672/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764218 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764193.341821:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764193.341821:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:2.0:1787764193.341822:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:2.0:1787764193.341824:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:2.0:1787764193.341824:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:2.0:1787764193.341825:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:2.0:1787764193.341826:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:2.0:1787764193.341827:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:2.0:1787764193.341828:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:2.0:1787764193.341829:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:2.0:1787764193.341830:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930780.
00000100:00000001:2.0:1787764193.341831:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764193.341832:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.341833:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.341836:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764193.341837:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764193.341838:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764193.341839:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764193.980263:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764193.980270:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764193.980270:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764193.980274:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764193.980276:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764193.980283:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764193.980285:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764193.980292:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764193.980292:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764193.980300:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764193.980302:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764193.980305:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764193.980305:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764193.980313:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764193.980317:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764193.980352:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764193.980356:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764193.980360:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764194.364285:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764194.364294:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764194.364296:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764194.364302:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764194.364306:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764194.364310:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764194.364312:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764194.364315:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764194.364318:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764194.620220:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316581619, at 4316581632
00000100:00000010:3.0:1787764194.620232:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7660.
00000100:00000040:2.0:1787764194.620267:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316581619, at 4316581632
00000100:00000010:2.0:1787764194.620277:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7b40.
00010000:00000001:3.0:1787764195.004282:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764195.004290:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764195.004291:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764195.004298:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764195.004300:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764195.004301:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764195.004305:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764195.004307:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764195.004313:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764195.004316:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764195.004317:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764195.004319:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764195.004321:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764195.004325:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764195.004328:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764195.004389:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764195.004394:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764195.004398:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764195.388257:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764195.388267:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764195.388270:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764195.388275:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764195.388280:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764195.388283:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764195.388286:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764195.388289:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764195.388292:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764196.028250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764196.028252:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764196.028259:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764196.028259:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764196.028262:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764196.028263:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764196.028270:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764196.028271:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764196.028276:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764196.028278:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764196.028279:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764196.028284:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764196.028286:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764196.028289:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764196.028292:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764196.028295:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764196.028297:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764196.028298:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764196.028301:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764196.028302:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764196.028309:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764196.028342:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764196.028347:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764196.028352:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764196.412306:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764196.412316:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764196.412319:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764196.412328:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764196.412332:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764196.412336:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764196.412338:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764196.412342:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764196.412345:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764197.052241:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764197.052242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764197.052247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764197.052249:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764197.052253:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764197.052256:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764197.052259:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764197.052260:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764197.052263:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764197.052265:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764197.052270:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764197.052272:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764197.052275:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764197.052277:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764197.052280:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764197.052349:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764197.052354:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764197.052357:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764197.436307:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764197.436325:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764197.436333:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764197.436340:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764197.436345:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764197.436348:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764197.436352:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764197.436355:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764197.436358:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764198.076236:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764198.076236:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764198.076242:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764198.076244:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764198.076248:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764198.076250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764198.076253:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764198.076255:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764198.076257:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764198.076258:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764198.076262:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764198.076263:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764198.076267:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764198.076268:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764198.076271:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764198.076271:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764198.076277:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764198.076284:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764198.076287:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764198.076290:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764198.076291:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764198.076345:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764198.076348:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764198.076351:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787764198.460273:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787764198.460280:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764198.460305:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764198.460309:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764198.460313:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00100000:1.0:1787764198.460315:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764198.460317:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764198.460323:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764198.460324:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764198.460330:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764198.460331:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1787764198.460337:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:2.0:1787764198.460341:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21913
00000100:00000001:1.0:1787764198.460342:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.460348:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764198.460352:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764198.460362:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764198.460366:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764198.460369:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764198.460374:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764198.460377:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764198.460380:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764198.460385:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764198.460387:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787764198.460390:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764198.460395:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787764198.460398:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787764198.460402:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787764198.460405:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764198.460407:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764198.460409:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764198.460412:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764198.460415:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787764198.460419:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764198.460421:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764198.460425:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764198.460429:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764198.460432:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764198.460434:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764198.460437:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764198.460445:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000020:00000040:2.0:1787764198.460449:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787764198.460454:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764198.460456:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764198.460459:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764198.460463:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764198.460467:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764198.460476:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5b8c00.
00000100:00000001:2.0:1787764198.460484:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764198.460495:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832700800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764198.460507:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764198.460513:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764198.460519:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832700800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764198.460552:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764198.460557:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764198.460560:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787764198.460562:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764198.460564:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764198.460567:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764198.460571:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21913
00000400:00000200:2.0:1787764198.460575:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764198.460578:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764198.460581:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764198.460584:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764198.460586:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764198.460588:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764198.460590:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764198.460592:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764198.460593:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:2.0:1787764198.460595:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764198.460598:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:2.0:1787764198.460600:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:2.0:1787764198.460602:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:2.0:1787764198.460605:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764198.460607:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764198.460609:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764198.460611:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764198.460614:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:2.0:1787764198.460617:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764198.460619:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764198.460621:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764198.460633:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764198.460636:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764198.460638:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764198.460642:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787764198.460646:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764198.460649:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764198.460656:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000020:00000040:2.0:1787764198.460659:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764198.460662:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764198.460664:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764198.460666:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764198.460668:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764198.460670:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764198.460676:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5b9800.
00000100:00000001:2.0:1787764198.460680:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764198.460690:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832700928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764198.460698:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764198.460701:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764198.460706:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832700928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764198.460716:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764198.460718:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764198.460723:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1787764198.460724:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764198.460726:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764198.460729:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764198.460733:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21913
00000400:00000200:2.0:1787764198.460738:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764198.460743:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764198.460745:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764198.460749:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764198.460751:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764198.460754:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764198.460756:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764198.460759:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764198.460762:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787764198.460764:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764198.460766:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:2.0:1787764198.460769:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:2.0:1787764198.460771:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:2.0:1787764198.460775:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764198.460777:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764198.460780:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764198.460782:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764198.460785:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:2.0:1787764198.460789:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764198.460791:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764198.460793:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787764198.460794:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:2.0:1787764198.460796:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764198.460801:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
02000000:00000001:2.0:1787764198.460801:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764198.460806:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764198.460807:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000008:00100000:2.0:1787764198.460809:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787764198.460813:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764198.460818:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832700800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764198.460818:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764198.460825:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000020:00000040:2.0:1787764198.460830:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000001:1.0:1787764198.460833:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1787764198.460836:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:1.0:1787764198.460837:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764198.460840:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764198.460844:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1787764198.460844:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764198.460846:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764198.460850:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764198.460850:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764198.460868:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764198.460873:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750da5bbc00.
00000100:00100000:1.0:1787764198.460875:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832700800:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764198.460879:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764198.460884:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00100000:2.0:1787764198.460889:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832701056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1787764198.460891:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764198.460895:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764198.460901:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1787764198.460904:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764198.460908:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764198.460910:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787764198.460915:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764198.460917:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832701056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787764198.460926:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764198.460934:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000100:00000001:2.0:1787764198.460936:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764198.460941:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:1.0:1787764198.460944:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000040:2.0:1787764198.460946:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:2.0:1787764198.460951:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:1.0:1787764198.460954:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832700800, portal 25
00000100:00080000:2.0:1787764198.460955:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787764198.460960:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764198.460960:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764198.460962:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764198.460964:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000001:2.0:1787764198.460968:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764198.460974:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21913
00000100:00000040:1.0:1787764198.460976:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832700800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764223 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764198.460983:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764198.460992:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1787764198.460993:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:2.0:1787764198.460997:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764198.461003:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000040:2.0:1787764198.461007:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000010:1.0:1787764198.461009:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000001:2.0:1787764198.461012:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:1.0:1787764198.461015:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832700800, offset 0
00000100:00000040:2.0:1787764198.461017:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764198.461023:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764198.461026:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764198.461027:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787764198.461031:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:1.0:1787764198.461032:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000020:00000040:2.0:1787764198.461035:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787764198.461039:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:2.0:1787764198.461042:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000200:1.0:1787764198.461044:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:2.0:1787764198.461050:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764198.461054:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787764198.461057:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000040:2.0:1787764198.461060:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764198.461065:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:1.0:1787764198.461066:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:2.0:1787764198.461070:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000400:00000200:1.0:1787764198.461075:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00080000:2.0:1787764198.461077:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764198.461081:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764198.461084:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
02000000:00000001:2.0:1787764198.461085:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787764198.461090:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764198.461096:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58710:58710:256:4294967295] 192.168.203.110@tcp LPNI seq info [58710:58710:8:4294967295]
02000000:00000001:2.0:1787764198.461098:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764198.461102:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764198.461106:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764198.461111:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000400:00000200:1.0:1787764198.461113:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000008:00000001:2.0:1787764198.461117:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764198.461121:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764198.461126:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
00000400:00000200:1.0:1787764198.461128:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000020:00000040:2.0:1787764198.461131:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000800:00000200:1.0:1787764198.461157:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764198.461164:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
02000000:00000001:2.0:1787764198.461166:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764198.461170:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764198.461174:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:1.0:1787764198.461175:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:2.0:1787764198.461179:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:1.0:1787764198.461182:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
02000000:00000001:2.0:1787764198.461183:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764198.461194:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751c3910c00.
00000800:00000200:1.1:1787764198.461195:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787764198.461202:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787764198.461203:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000100:00100000:2.0:1787764198.461212:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832701184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000100:1.0:1787764198.461213:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764198.461218:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764198.461226:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1787764198.461227:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764198.461233:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764198.461234:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:2.0:1787764198.461241:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832701184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787764198.461242:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764198.461248:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764198.461255:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:2.0:1787764198.461257:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787764198.461263:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832700800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764223 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787764198.461264:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21923)
00000100:00000001:1.0:1787764198.461277:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764198.461282:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832700800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764223 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.461290:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764198.461291:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764198.461298:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764198.461303:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764198.461306:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316586740
00000100:00000010:1.0:1787764198.461312:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4720.
00000100:00000001:1.0:1787764198.461318:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787764198.461325:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316586740
00000100:00000010:1.0:1787764198.461328:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c44e0.
00000100:00000001:1.0:1787764198.461331:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787764198.461334:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764198.461337:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764198.461340:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764198.461344:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764198/real 1787764198]  req@ffff9751f865ad00 x1874599832700800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764198.461354:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764198.461356:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764198.461358:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764198.461361:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764198.461363:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764198.461365:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764198.461369:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832700800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764198.461378:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832700800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.461385:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764198.461387:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.461389:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764198.461391:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764198.461396:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832700800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.461403:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764198.461408:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832700800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.461415:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764198.461419:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.461421:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764198.461422:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.461424:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764198.461426:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764198.461430:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764198.461434:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764198.461436:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764198.461441:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832700800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764198.461451:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832700800:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787764198.461456:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764198.461460:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832700928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.461467:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764198.461469:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764198.461471:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764198.461473:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764198.461476:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764198.461486:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832700928:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764198.461490:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764198.461493:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764198.461495:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764198.461496:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764198.461501:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787764198.461503:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764198.461507:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764198.461511:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764198.461515:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764198.461518:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832700928, portal 10
00000100:00000001:1.0:1787764198.461521:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764198.461523:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:1.0:1787764198.461529:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832700928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764223 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.461536:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764198.461539:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764198.461541:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764198.461544:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832700928, offset 0
00000400:00000200:1.0:1787764198.461549:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764198.461554:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764198.461558:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764198.461562:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764198.461565:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764198.461568:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764198.461571:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764198.461574:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58711:58711:256:4294967295] 192.168.203.110@tcp LPNI seq info [58711:58711:8:4294967295]
00000400:00000200:1.0:1787764198.461578:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764198.461585:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764198.461590:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764198.461594:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764198.461598:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764198.461602:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787764198.461606:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764198.461610:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787764198.461614:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764198.461616:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764198.461619:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764198.461623:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764198.461627:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764198.461629:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764198.461631:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764198.461635:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832700928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764223 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.461650:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764198.461654:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832700928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764223 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.461660:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764198.461662:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764198.461665:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764198.461670:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787764198.461673:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764198.461675:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316585504
00000100:00000001:1.0:1787764198.461678:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316586740
00000100:00000001:1.0:1787764198.461680:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316585504
00000100:00000001:1.0:1787764198.461681:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316586740
00000100:00000001:1.0:1787764198.461684:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764198.461686:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764198.461688:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764198.461692:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764198/real 1787764198]  req@ffff9751f86583c0 x1874599832700928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764198.461700:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764198.461701:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764198.461703:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764198.461705:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764198.461706:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764198.461708:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764198.461712:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832700928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764198.461720:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832700928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.461727:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764198.461729:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.461731:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764198.461732:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764198.461736:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832700928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.461742:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764198.461746:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832700928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.461754:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764198.461757:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.461759:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764198.461760:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.461762:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764198.461776:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764198.461780:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764198.461785:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764198.461788:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764198.461793:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832700928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764198.461803:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832700928:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764198.461808:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.461811:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764198.461815:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832700800/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.461820:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764198.461822:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764198.461824:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787764198.461827:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764198.461829:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764198.461831:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787764198.461833:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764198.461835:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b8c00.
02000000:00000001:1.0:1787764198.461839:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764198.461840:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764198.461842:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:1.0:1787764198.461846:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764198.461847:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.461849:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764198.461852:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832700928/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.461873:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764198.461874:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764198.461876:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787764198.461878:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764198.461879:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764198.461880:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764198.461882:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764198.461883:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5b9800.
02000000:00000001:1.0:1787764198.461885:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764198.461886:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764198.461887:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:1.0:1787764198.461889:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764198.461890:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.461891:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.461897:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764198.461899:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764198.461900:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764198.461901:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764198.461902:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764198.461905:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832701056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.461911:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764198.461912:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764198.461914:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764198.461915:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764198.461917:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764198.461920:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832701056:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764198.461922:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764198.461924:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764198.461925:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764198.461926:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764198.461929:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787764198.461930:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764198.461933:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764198.461935:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764198.461937:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764198.461940:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832701056, portal 10
00000100:00000001:1.0:1787764198.461941:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764198.461942:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787764198.461946:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832701056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764223 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.461951:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764198.461953:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764198.461955:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764198.461957:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832701056, offset 0
00000400:00000200:1.0:1787764198.461960:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764198.461963:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764198.461966:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764198.461968:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764198.461970:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764198.461972:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764198.461974:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764198.461976:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58712:58712:256:4294967295] 192.168.203.110@tcp LPNI seq info [58712:58712:8:4294967295]
00000400:00000200:1.0:1787764198.461979:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764198.461984:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764198.461987:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764198.461989:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764198.461992:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764198.461994:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787764198.461997:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764198.461999:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787764198.462002:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764198.462003:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764198.462005:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764198.462007:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764198.462009:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764198.462010:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764198.462012:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764198.462014:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832701056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764223 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.462019:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764198.462021:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832701056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764223 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.462026:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764198.462026:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764198.462028:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764198.462030:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787764198.462032:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764198.462034:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316585492
00000100:00000001:1.0:1787764198.462035:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316586740
00000100:00000001:1.0:1787764198.462036:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316585492
00000100:00000001:1.0:1787764198.462038:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316586740
00000100:00000001:1.0:1787764198.462039:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764198.462041:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764198.462042:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764198.462044:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764198/real 1787764198]  req@ffff9751f8659a40 x1874599832701056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764198.462054:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764198.462054:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764198.462056:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764198.462057:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764198.462058:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764198.462059:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764198.462061:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832701056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764198.462066:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832701056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.462070:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764198.462071:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.462073:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764198.462074:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764198.462076:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832701056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.462080:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764198.462082:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832701056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.462086:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764198.462088:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.462089:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764198.462089:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.462091:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764198.462092:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764198.462093:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764198.462095:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764198.462096:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764198.462099:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832701056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764198.462104:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832701056:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764198.462107:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764198.462109:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832701184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.462113:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764198.462114:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764198.462115:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764198.462116:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764198.462117:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764198.462119:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832701184:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764198.462121:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764198.462122:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764198.462123:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764198.462124:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764198.462131:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787764198.462132:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764198.462153:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764198.462155:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764198.462159:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764198.462161:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832701184, portal 4
00000100:00000001:1.0:1787764198.462163:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764198.462164:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787764198.462169:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832701184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764223 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.462173:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764198.462175:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764198.462176:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764198.462178:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832701184, offset 0
00000400:00000200:1.0:1787764198.462181:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764198.462183:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764198.462186:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764198.462188:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764198.462191:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764198.462194:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764198.462196:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764198.462199:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58713:58713:256:4294967295] 192.168.203.110@tcp LPNI seq info [58713:58713:8:4294967295]
00000400:00000200:1.0:1787764198.462204:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764198.462210:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764198.462215:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764198.462219:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764198.462224:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764198.462227:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:1.1:1787764198.462231:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764198.462235:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:1.0:1787764198.462239:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764198.462241:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764198.462244:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764198.462247:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764198.462249:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764198.462250:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764198.462252:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764198.462255:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832701184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764223 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.462260:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764198.462263:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832701184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764223 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.462267:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764198.462268:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764198.462271:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764198.462273:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787764198.462275:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764198.462277:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316585510
00000100:00000001:1.0:1787764198.462279:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316586740
00000100:00000001:1.0:1787764198.462281:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316585510
00000100:00000001:1.0:1787764198.462282:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316586740
00000100:00000001:1.0:1787764198.462283:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764198.462285:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764198.462286:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764198.462288:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764198/real 1787764198]  req@ffff9751f8658000 x1874599832701184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764198.462293:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764198.462294:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764198.462296:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764198.462297:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764198.462298:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764198.462299:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764198.462301:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832701184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764198.462307:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832701184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.462310:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764198.462311:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.462312:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764198.462313:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764198.462315:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832701184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.462319:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764198.462322:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832701184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.462326:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764198.462328:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.462329:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764198.462330:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.462331:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764198.462332:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764198.462334:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764198.462335:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764198.462337:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764198.462339:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832701184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764198.462344:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832701184:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764198.462346:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.462347:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764198.462349:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832701056/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.462353:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764198.462354:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764198.462355:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787764198.462356:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764198.462357:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764198.462358:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764198.462359:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764198.462360:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750da5bbc00.
02000000:00000001:1.0:1787764198.462362:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764198.462363:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764198.462364:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787764198.462365:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764198.462366:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.462367:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764198.462369:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832701184/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764223 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764198.462377:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764198.462378:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764198.462379:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787764198.462380:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764198.462381:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764198.462381:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787764198.462382:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764198.462383:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751c3910c00.
02000000:00000001:1.0:1787764198.462386:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764198.462387:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764198.462388:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:1.0:1787764198.462389:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764198.462390:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.462391:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764198.462393:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764198.462394:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764198.462395:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764198.462396:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764199.100207:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764199.100231:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764199.100232:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764199.100233:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764199.100237:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764199.100240:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764199.100243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764199.100245:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764199.100247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764199.100251:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764199.100252:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764199.100256:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764199.100257:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764199.100261:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764199.100265:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764199.100322:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764199.100327:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764199.100332:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764199.484258:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764199.484268:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764199.484270:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764199.484277:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764199.484294:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764199.484299:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764199.484301:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764199.484304:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764199.484306:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764199.740181:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316586740, at 4316586752
00000100:00000010:3.0:1787764199.740193:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c44e0.
00000100:00000040:2.0:1787764199.740233:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316586740, at 4316586752
00000100:00000010:2.0:1787764199.740241:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4720.
00000100:00000001:2.0:1787764200.124214:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764200.124218:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764200.124223:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764200.124228:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764200.124230:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764200.124232:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764200.124233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764200.124236:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764200.124237:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764200.124248:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764200.124253:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764200.124257:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764200.124260:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764200.124263:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764200.124266:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764200.124284:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764200.124288:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764200.124291:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764200.508242:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764200.508249:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764200.508251:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764200.508256:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764200.508259:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764200.508262:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764200.508264:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764200.508267:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764200.508269:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764201.148218:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764201.148222:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764201.148224:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764201.148226:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764201.148228:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764201.148234:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764201.148235:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764201.148238:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764201.148239:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764201.148243:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764201.148243:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764201.148246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764201.148247:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764201.148250:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764201.148252:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764201.148293:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764201.148298:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764201.148301:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764201.532217:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764201.532226:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764201.532228:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764201.532234:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764201.532239:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764201.532243:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764201.532246:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764201.532249:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764201.532252:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764202.172213:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764202.172222:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764202.172225:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764202.172230:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764202.172235:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764202.172240:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764202.172243:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764202.172246:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764202.172248:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764202.172254:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764202.172261:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764202.172269:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764202.172270:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764202.172277:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764202.172279:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764202.172332:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764202.172335:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764202.172338:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764202.556198:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764202.556207:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764202.556210:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764202.556217:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764202.556222:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764202.556226:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764202.556228:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764202.556231:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764202.556234:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764203.196230:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764203.196233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764203.196237:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764203.196238:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764203.196240:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764203.196244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764203.196245:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764203.196246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764203.196250:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764203.196251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764203.196255:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764203.196256:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764203.196260:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764203.196260:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764203.196264:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764203.196265:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764203.196267:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764203.196269:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764203.196271:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764203.196273:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764203.196275:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764203.196322:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764203.196326:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764203.196328:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764203.580237:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:2.0:1787764203.580240:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787764203.580245:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:2.0:1787764203.580248:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787764203.580249:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764203.580251:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:1.0:1787764203.580255:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:2.0:1787764203.580256:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764203.580262:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764203.580263:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1787764203.580268:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764203.580273:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:2.0:1787764203.580273:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21918
00000100:00000001:1.0:1787764203.580279:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:2.0:1787764203.580284:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764203.580285:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764203.580293:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764203.580297:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764203.580300:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764203.580303:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764203.580305:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764203.580307:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764203.580310:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764203.580312:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787764203.580314:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764203.580317:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787764203.580319:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787764203.580321:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787764203.580323:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764203.580324:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764203.580325:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764203.580327:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764203.580329:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787764203.580331:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764203.580332:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764203.580334:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764203.580337:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764203.580339:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764203.580341:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764203.580343:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764203.580351:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658780.
00000020:00000040:2.0:1787764203.580353:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787764203.580356:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764203.580357:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764203.580359:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764203.580361:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764203.580364:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764203.580371:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751c3913000.
00000100:00000001:2.0:1787764203.580376:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764203.580384:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658780 x1874599832701312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764203.580392:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764203.580396:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764203.580399:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000744875b5] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658780 x1874599832701312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764203.580417:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764203.580419:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764203.580422:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787764203.580423:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764203.580424:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764203.580425:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764203.580427:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21918
00000400:00000200:2.0:1787764203.580429:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764203.580432:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764203.580433:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764203.580436:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764203.580438:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764203.580440:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764203.580442:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764203.580444:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764203.580445:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00000001:1.0:1787764203.580447:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:2.0:1787764203.580447:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764203.580451:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:2.0:1787764203.580453:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:1.0:1787764203.580454:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1787764203.580455:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:1.0:1787764203.580458:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764203.580459:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764203.580463:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764203.580466:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764203.580469:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:1.0:1787764203.580470:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832701312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787764203.580471:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:2.0:1787764203.580478:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764203.580483:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764203.580485:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764203.580490:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764203.580492:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1787764203.580493:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764203.580495:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764203.580497:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1787764203.580500:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000008:00100000:2.0:1787764203.580501:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:1.0:1787764203.580504:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000008:00000001:2.0:1787764203.580505:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764203.580510:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764203.580510:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764203.580515:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
00000020:00000040:2.0:1787764203.580516:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764203.580518:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764203.580520:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764203.580524:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832701312:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764203.580524:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764203.580527:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764203.580531:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764203.580532:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000010:2.0:1787764203.580536:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751c3910800.
02000000:00000001:1.0:1787764203.580537:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764203.580541:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764203.580541:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764203.580543:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00100000:2.0:1787764203.580549:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832701440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000010:1.0:1787764203.580550:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787764203.580555:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764203.580563:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1787764203.580565:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764203.580566:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787764203.580570:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000100:00000040:2.0:1787764203.580571:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832701440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1787764203.580577:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000001:2.0:1787764203.580579:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764203.580582:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:1.0:1787764203.580584:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832701312, portal 25
00000100:00080000:2.0:1787764203.580586:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1787764203.580589:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764203.580590:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764203.580592:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1787764203.580594:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00080000:2.0:1787764203.580597:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21918
00000100:00000001:1.0:1787764203.580598:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000400:00000200:2.0:1787764203.580609:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:1.0:1787764203.580610:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832701312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764203.580619:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1787764203.580620:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764203.580626:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00080000:2.0:1787764203.580626:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000010:1.0:1787764203.580630:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000040:2.0:1787764203.580634:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000200:1.0:1787764203.580635:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832701312, offset 0
00000100:00000001:2.0:1787764203.580641:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787764203.580646:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000040:2.0:1787764203.580648:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764203.580652:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764203.580656:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:1.0:1787764203.580657:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:2.0:1787764203.580660:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787764203.580664:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:1.0:1787764203.580667:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000020:00000040:2.0:1787764203.580669:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:2.0:1787764203.580673:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:1.0:1787764203.580674:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000001:2.0:1787764203.580679:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:1.0:1787764203.580681:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000040:2.0:1787764203.580686:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:1.0:1787764203.580690:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1787764203.580690:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764203.580695:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:1.0:1787764203.580700:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:2.0:1787764203.580701:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764203.580706:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:1.0:1787764203.580710:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58714:58714:256:4294967295] 192.168.203.110@tcp LPNI seq info [58714:58714:8:4294967295]
00000100:00080000:2.0:1787764203.580713:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764203.580720:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764203.580723:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:2.0:1787764203.580724:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764203.580730:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764203.580734:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764203.580737:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:1.0:1787764203.580741:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000008:00100000:2.0:1787764203.580742:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787764203.580749:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787764203.580751:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787764203.580754:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1787764203.580757:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000100:00000010:2.0:1787764203.580761:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000800:00000200:1.0:1787764203.580767:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:2.0:1787764203.580768:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000800:00000200:1.0:1787764203.580770:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
02000000:00000001:2.0:1787764203.580773:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764203.580777:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787764203.580777:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787764203.580782:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:1.1:1787764203.580783:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
02000000:00000001:2.0:1787764203.580787:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000100:1.0:1787764203.580791:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
02000000:00000001:2.0:1787764203.580793:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1787764203.580794:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764203.580802:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000010:2.0:1787764203.580803:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751c3910c00.
00000400:00000200:1.0:1787764203.580808:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1787764203.580810:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764203.580812:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764203.580816:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764203.580833:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00100000:2.0:1787764203.580836:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832701568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:1.0:1787764203.580840:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832701312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.580866:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787764203.580867:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:1.0:1787764203.580872:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832701312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764228 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764203.580873:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764203.580883:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764203.580885:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:2.0:1787764203.580885:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832701568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.580893:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764203.580897:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764203.580899:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764203.580904:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316591859
00000100:00000040:2.0:1787764203.580904:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000010:1.0:1787764203.580911:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4de0.
00000100:00000040:2.0:1787764203.580912:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:2.0:1787764203.580917:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787764203.580919:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00080000:2.0:1787764203.580923:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787764203.580927:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316591859
00000100:00000001:2.0:1787764203.580928:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000010:1.0:1787764203.580933:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4060.
00000100:00080000:2.0:1787764203.580933:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764203.580938:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787764203.580940:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1787764203.580942:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764203.580946:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764203.580947:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21918
00000100:00000001:1.0:1787764203.580950:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764203.580957:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764203/real 1787764203]  req@ffff9751f8658780 x1874599832701312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764203.580958:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000001:1.0:1787764203.580968:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000200:2.0:1787764203.580968:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000001:1.0:1787764203.580972:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00080000:2.0:1787764203.580974:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000010:1.0:1787764203.580975:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764203.580979:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764203.580982:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000040:2.0:1787764203.580983:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764203.580985:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:2.0:1787764203.580989:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764203.580993:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764203.580997:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000200:1.0:1787764203.580999:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832701312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787764203.581002:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764203.581006:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787764203.581012:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000200:1.0:1787764203.581013:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832701312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787764203.581017:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000001:1.0:1787764203.581022:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764203.581024:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1787764203.581024:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:1.0:1787764203.581028:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764203.581031:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1787764203.581031:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00100000:1.0:1787764203.581038:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832701312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787764203.581041:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764203.581047:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787764203.581049:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764203.581054:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000040:1.0:1787764203.581056:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832701312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764203.581061:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787764203.581066:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000040:2.0:1787764203.581068:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:1.0:1787764203.581070:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764203.581073:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764203.581075:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1787764203.581077:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764203.581078:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764203.581081:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764203.581083:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764203.581084:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000001:2.0:1787764203.581090:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00080000:1.0:1787764203.581092:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764203.581095:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
02000000:00000001:2.0:1787764203.581096:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1787764203.581099:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832701312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764203.581102:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764203.581107:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764203.581113:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00100000:1.0:1787764203.581114:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832701312:192.168.203.110@tcp:250:kworker.0
00000008:00000001:2.0:1787764203.581120:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764203.581123:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764203.581125:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764203.581128:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764203.581156:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832701312/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1787764203.581163:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000100:00000001:1.0:1787764203.581166:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
00000020:00000040:2.0:1787764203.581170:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787764203.581171:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764203.581175:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:2.0:1787764203.581177:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764203.581180:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
02000000:00000001:2.0:1787764203.581181:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764203.581183:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764203.581186:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000100:00000001:2.0:1787764203.581187:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000020:00000001:1.0:1787764203.581188:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764203.581192:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751c3913000.
02000000:00000001:2.0:1787764203.581192:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764203.581196:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764203.581198:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764203.581202:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764203.581203:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
02000000:00000010:2.0:1787764203.581205:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751c3913000.
00000100:00000001:1.0:1787764203.581206:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764203.581208:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764203.581211:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764203.581213:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764203.581218:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764203.581222:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764203.581224:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832701696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764203.581225:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764203.581227:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764203.581230:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764203.581237:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832701440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764203.581242:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764203.581246:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764203.581248:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764203.581249:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764203.581255:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764203.581257:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764203.581260:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764203.581262:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832701696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764203.581264:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832701440:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764203.581271:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764203.581274:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764203.581277:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764203.581283:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1787764203.581286:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764203.581288:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787764203.581292:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764203.581294:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21928)
00000400:00000200:1.0:1787764203.581300:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764203.581305:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764203.581309:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764203.581314:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832701440, portal 10
00000100:00000001:1.0:1787764203.581316:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764203.581318:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:1.0:1787764203.581326:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832701440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.581337:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764203.581339:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764203.581341:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764203.581344:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832701440, offset 0
00000400:00000200:1.0:1787764203.581349:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764203.581353:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764203.581357:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764203.581360:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764203.581363:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764203.581366:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764203.581369:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764203.581372:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58715:58715:256:4294967295] 192.168.203.110@tcp LPNI seq info [58715:58715:8:4294967295]
00000400:00000200:1.0:1787764203.581377:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764203.581385:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764203.581389:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764203.581393:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764203.581396:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764203.581398:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787764203.581401:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764203.581403:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787764203.581405:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764203.581407:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764203.581409:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764203.581411:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764203.581414:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764203.581416:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764203.581418:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764203.581422:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832701440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.581431:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764203.581434:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832701440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764228 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.581442:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764203.581444:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764203.581447:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764203.581452:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787764203.581456:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764203.581458:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316590624
00000100:00000001:1.0:1787764203.581461:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316591859
00000100:00000001:1.0:1787764203.581463:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316590624
00000100:00000001:1.0:1787764203.581465:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316591859
00000100:00000001:1.0:1787764203.581467:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764203.581469:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764203.581470:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764203.581473:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764203/real 1787764203]  req@ffff9751f8658000 x1874599832701440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764203.581479:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764203.581480:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764203.581481:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764203.581483:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764203.581484:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764203.581485:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764203.581487:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832701440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764203.581493:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832701440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.581497:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764203.581498:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764203.581499:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764203.581500:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764203.581502:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832701440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.581506:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764203.581508:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832701440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.581513:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764203.581514:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764203.581515:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764203.581516:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764203.581517:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764203.581527:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764203.581529:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764203.581531:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764203.581533:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764203.581535:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832701440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764203.581540:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832701440:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764203.581542:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764203.581545:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832701568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.581549:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764203.581549:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764203.581551:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764203.581552:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764203.581553:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764203.581555:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832701568:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764203.581557:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764203.581559:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764203.581560:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764203.581561:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764203.581564:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787764203.581565:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764203.581567:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764203.581569:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764203.581571:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764203.581573:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832701568, portal 10
00000100:00000001:1.0:1787764203.581574:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764203.581575:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:1.0:1787764203.581579:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832701568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.581588:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764203.581589:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764203.581590:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764203.581592:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832701568, offset 0
00000400:00000200:1.0:1787764203.581595:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764203.581597:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764203.581599:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764203.581601:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764203.581603:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764203.581605:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764203.581607:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764203.581609:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58716:58716:256:4294967295] 192.168.203.110@tcp LPNI seq info [58716:58716:8:4294967295]
00000400:00000200:1.0:1787764203.581611:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764203.581616:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764203.581618:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764203.581620:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764203.581623:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764203.581625:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787764203.581628:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764203.581629:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787764203.581632:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764203.581633:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764203.581636:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764203.581638:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764203.581640:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764203.581641:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764203.581642:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764203.581645:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832701568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.581650:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764203.581652:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832701568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764228 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.581660:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764203.581662:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764203.581663:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764203.581665:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787764203.581667:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764203.581669:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316590612
00000100:00000001:1.0:1787764203.581670:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316591859
00000100:00000001:1.0:1787764203.581672:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316590612
00000100:00000001:1.0:1787764203.581673:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316591859
00000100:00000001:1.0:1787764203.581674:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764203.581675:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764203.581676:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764203.581679:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764203/real 1787764203]  req@ffff9751f8659a40 x1874599832701568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764203.581686:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764203.581687:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764203.581689:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764203.581691:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764203.581693:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764203.581695:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764203.581698:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832701568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764203.581707:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832701568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.581714:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764203.581715:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764203.581717:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764203.581719:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764203.581723:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832701568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.581731:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764203.581735:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832701568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.581740:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764203.581742:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764203.581743:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764203.581744:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764203.581746:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764203.581747:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764203.581748:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764203.581750:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764203.581751:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764203.581754:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832701568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764203.581759:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832701568:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764203.581761:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764203.581763:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764203.581765:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832701440/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.581769:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764203.581770:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764203.581771:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787764203.581773:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764203.581774:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764203.581775:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764203.581777:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764203.581779:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751c3910800.
02000000:00000001:1.0:1787764203.581784:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764203.581785:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764203.581787:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:1.0:1787764203.581789:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764203.581791:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764203.581793:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764203.581796:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832701568/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.581802:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764203.581804:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764203.581805:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787764203.581808:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764203.581809:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764203.581810:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764203.581812:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764203.581813:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751c3910c00.
02000000:00000001:1.0:1787764203.581816:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764203.581817:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764203.581819:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:1.0:1787764203.581836:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764203.581838:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764203.581840:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764203.581845:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764203.581847:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764203.581849:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764203.581850:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764203.581852:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764203.581856:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832701696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.581863:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764203.581865:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764203.581867:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764203.581869:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764203.581871:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764203.581874:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832701696:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764203.581880:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764203.581882:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764203.581884:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764203.581886:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764203.581889:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787764203.581892:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764203.581896:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764203.581899:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764203.581903:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764203.581906:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832701696, portal 4
00000100:00000001:1.0:1787764203.581908:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764203.581909:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:1.0:1787764203.581915:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832701696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.581920:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764203.581922:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764203.581923:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764203.581925:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832701696, offset 0
00000400:00000200:1.0:1787764203.581928:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764203.581931:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764203.581934:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764203.581936:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764203.581938:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764203.581940:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764203.581941:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764203.581943:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58717:58717:256:4294967295] 192.168.203.110@tcp LPNI seq info [58717:58717:8:4294967295]
00000400:00000200:1.0:1787764203.581946:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764203.581950:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764203.581954:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764203.581956:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764203.581958:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764203.581961:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:1.1:1787764203.581964:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764203.581966:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:1.0:1787764203.581969:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764203.581971:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764203.581973:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764203.581975:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764203.581977:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764203.581978:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764203.581980:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764203.581982:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832701696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.581988:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764203.581990:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832701696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764228 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.581994:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764203.581995:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764203.581997:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764203.581999:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787764203.582001:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764203.582003:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316590629
00000100:00000001:1.0:1787764203.582005:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316591859
00000100:00000001:1.0:1787764203.582006:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316590629
00000100:00000001:1.0:1787764203.582008:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316591859
00000100:00000001:1.0:1787764203.582009:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764203.582010:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764203.582012:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764203.582014:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764203/real 1787764203]  req@ffff9751f86583c0 x1874599832701696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764203.582018:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764203.582019:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764203.582021:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764203.582022:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764203.582023:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764203.582025:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764203.582027:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832701696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764203.582032:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832701696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.582036:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764203.582037:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764203.582038:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764203.582039:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764203.582041:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832701696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.582045:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764203.582047:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832701696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.582052:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764203.582054:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764203.582055:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764203.582055:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764203.582057:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764203.582057:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764203.582059:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764203.582061:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764203.582062:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764203.582064:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832701696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764203.582069:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832701696:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764203.582072:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764203.582073:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764203.582075:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832701696/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764228 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764203.582079:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764203.582080:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764203.582081:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787764203.582083:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764203.582083:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764203.582084:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787764203.582085:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764203.582086:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751c3913000.
02000000:00000001:1.0:1787764203.582088:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764203.582089:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764203.582090:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:1.0:1787764203.582091:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764203.582092:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764203.582093:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764203.582096:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764203.582097:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764203.582098:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764203.582099:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764204.220224:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764204.220228:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764204.220231:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764204.220234:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764204.220234:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764204.220241:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764204.220244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764204.220245:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764204.220250:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764204.220251:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764204.220255:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764204.220258:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764204.220258:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764204.220263:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764204.220265:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764204.220323:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764204.220327:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764204.220331:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764204.604239:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764204.604246:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764204.604248:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764204.604253:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764204.604256:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764204.604259:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764204.604261:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764204.604264:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764204.604267:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764204.860187:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316591859, at 4316591872
00000100:00000010:3.0:1787764204.860204:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4060.
00000100:00000040:2.0:1787764204.860280:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316591859, at 4316591872
00000100:00000010:2.0:1787764204.860288:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4de0.
00000100:00000001:2.0:1787764205.244249:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764205.244261:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764205.244273:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764205.244280:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764205.244280:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764205.244290:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764205.244292:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764205.244298:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764205.244299:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764205.244303:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764205.244304:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764205.244308:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764205.244312:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764205.244315:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764205.244316:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764205.244387:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764205.244391:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764205.244394:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764205.628229:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764205.628236:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764205.628238:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764205.628244:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764205.628248:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764205.628251:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764205.628253:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764205.628255:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764205.628258:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764206.268217:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764206.268225:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764206.268227:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764206.268228:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764206.268233:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764206.268235:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764206.268239:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764206.268239:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764206.268246:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764206.268247:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764206.268250:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764206.268253:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764206.268254:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764206.268258:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764206.268260:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764206.268261:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764206.268269:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764206.268270:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764206.268273:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764206.268277:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764206.268279:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764206.268336:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764206.268340:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764206.268343:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764206.652245:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764206.652261:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764206.652263:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764206.652267:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764206.652270:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764206.652273:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764206.652275:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764206.652277:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764206.652279:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764207.292256:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764207.292261:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764207.292265:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764207.292269:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764207.292271:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764207.292273:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764207.292275:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764207.292277:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764207.292278:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764207.292280:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764207.292283:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764207.292284:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764207.292316:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764207.292321:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764207.292324:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764207.292328:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:3.0:1787764207.292333:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764207.292337:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:1.0:1787764207.292340:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000100:00000001:3.0:1787764207.292342:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:1.0:1787764207.292344:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764207.292348:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00010000:1.0:1787764207.292350:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764207.292354:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764207.676201:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764207.676209:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764207.676212:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764207.676218:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764207.676221:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764207.676224:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764207.676226:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764207.676230:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764207.676233:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764208.316230:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764208.316235:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764208.316240:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764208.316240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764208.316244:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764208.316245:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764208.316249:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764208.316249:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764208.316253:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764208.316255:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787764208.316259:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764208.316261:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764208.316263:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764208.316265:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764208.316267:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764208.316296:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764208.316299:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764208.316302:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764208.700256:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764208.700264:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764208.700267:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764208.700272:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764208.700276:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764208.700279:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764208.700281:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764208.700284:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764208.700287:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764208.700379:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764208.700386:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787764208.700388:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764208.700391:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764208.700394:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764208.700400:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21923
00000400:00000200:0.0:1787764208.700406:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764208.700411:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764208.700414:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764208.700416:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764208.700418:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764208.700420:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764208.700422:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764208.700425:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764208.700426:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764208.700428:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764208.700431:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764208.700433:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764208.700434:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787764208.700436:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764208.700438:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764208.700439:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764208.700441:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764208.700443:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787764208.700446:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764208.700448:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764208.700450:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764208.700453:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764208.700455:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764208.700456:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764208.700458:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764208.700466:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000020:00000040:0.0:1787764208.700468:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764208.700471:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764208.700472:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764208.700474:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764208.700476:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764208.700480:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764208.700486:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787764208.700491:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764208.700499:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832701824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764208.700507:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764208.700510:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764208.700514:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832701824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764208.700543:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764208.700547:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764208.700551:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764208.700552:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764208.700554:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764208.700557:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764208.700561:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21923
00000400:00000200:0.0:1787764208.700566:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764208.700570:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1787764208.700571:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:0.0:1787764208.700573:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787764208.700575:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764208.700578:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787764208.700580:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764208.700585:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764208.700586:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876de00 x1874599832701824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764208.700593:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764208.700600:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764208.700601:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787764208.700602:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764208.700608:0:279196:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:1.0:1787764208.700609:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:0.0:1787764208.700610:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
02000000:00000001:1.0:1787764208.700612:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:0.0:1787764208.700613:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764208.700618:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
02000000:00000001:1.0:1787764208.700618:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764208.700624:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00100000:1.0:1787764208.700626:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832701824:192.168.203.110@tcp:250:kworker.0
00000020:00000001:0.0:1787764208.700627:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:1.0:1787764208.700634:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:0.0:1787764208.700637:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:1.0:1787764208.700639:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1787764208.700640:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1787764208.700642:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764208.700643:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:1.0:1787764208.700646:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1787764208.700647:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764208.700650:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000010:1.0:1787764208.700652:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
00000100:00080000:0.0:1787764208.700653:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:1.0:1787764208.700656:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764208.700657:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764208.700660:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787764208.700663:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787764208.700665:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:1.0:1787764208.700667:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
02000000:00000001:0.0:1787764208.700668:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764208.700672:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:1.0:1787764208.700676:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000008:00100000:0.0:1787764208.700677:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787764208.700681:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764208.700683:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787764208.700685:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832701824, portal 25
00000100:00000010:0.0:1787764208.700688:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
00000100:00000001:1.0:1787764208.700688:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764208.700691:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049525760 : -115096660025856 : ffff9751f876de00)
00000020:00000040:0.0:1787764208.700692:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787764208.700696:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:1.0:1787764208.700698:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876de00 x1874599832701824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764233 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787764208.700699:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764208.700703:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764208.700705:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764208.700707:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764208.700713:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:1.0:1787764208.700715:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1787764208.700717:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764208.700722:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00100000:0.0:1787764208.700726:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832701952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1787764208.700729:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000001:0.0:1787764208.700736:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:1.0:1787764208.700737:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832701824, offset 0
00000100:00000001:0.0:1787764208.700738:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764208.700745:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832701952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787764208.700752:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787764208.700760:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764208.700764:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787764208.700765:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:0.0:1787764208.700771:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:1.0:1787764208.700774:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:0.0:1787764208.700776:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764208.700780:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1787764208.700780:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764208.700799:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:0.0:1787764208.700804:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1787764208.700806:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00080000:0.0:1787764208.700811:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21923
00000400:00000200:1.0:1787764208.700811:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764208.700817:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58718:58718:256:4294967295] 192.168.203.110@tcp LPNI seq info [58718:58718:8:4294967295]
00000400:00000200:1.0:1787764208.700821:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764208.700826:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764208.700832:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764208.700837:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1787764208.700837:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:0.0:1787764208.700844:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764208.700849:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.0:1787764208.700849:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787764208.700854:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000010:1.0:1787764208.700856:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000100:00000001:0.0:1787764208.700857:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764208.700861:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764208.700863:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787764208.700864:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764208.700868:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000200:1.0:1787764208.700868:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787764208.700873:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:1.0:1787764208.700874:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000020:00000001:0.0:1787764208.700876:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787764208.700883:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764208.700887:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.1:1787764208.700888:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787764208.700891:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764208.700895:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:1.1:1787764208.700896:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000020:00000040:0.0:1787764208.700897:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787764208.700902:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764208.700906:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:1.0:1787764208.700907:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
02000000:00000001:0.0:1787764208.700908:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764208.700912:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000800:00000010:1.0:1787764208.700914:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
02000000:00000001:0.0:1787764208.700916:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764208.700919:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764208.700920:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787764208.700922:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764208.700922:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1787764208.700924:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764208.700930:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000400:00000200:1.0:1787764208.700932:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000020:00000040:0.0:1787764208.700935:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000400:00000200:1.0:1787764208.700938:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
02000000:00000001:0.0:1787764208.700939:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:1.0:1787764208.700942:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
02000000:00000001:0.0:1787764208.700943:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764208.700948:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764208.700949:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1787764208.700950:0:257575:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:0.0:1787764208.700951:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764208.700956:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000200:1.0:1787764208.700957:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876de00 x1874599832701824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764233 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764208.700961:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764208.700970:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832702080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764208.700980:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764208.700980:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1787764208.700984:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764208.700989:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876de00 x1874599832701824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764233 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764208.700990:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832702080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.701014:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764208.701016:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787764208.701022:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764208.701024:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764208.701025:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787764208.701029:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:1.0:1787764208.701031:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:0.0:1787764208.701033:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787764208.701034:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316596979
00000100:00080000:0.0:1787764208.701040:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787764208.701042:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000010:1.0:1787764208.701043:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c43c0.
00000100:00080000:0.0:1787764208.701044:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764208.701048:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764208.701050:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21923
00000100:00000001:1.0:1787764208.701051:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000400:00000200:0.0:1787764208.701055:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764208.701059:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1787764208.701060:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316596979
00000100:00080000:0.0:1787764208.701061:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000010:1.0:1787764208.701066:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4a20.
00000100:00000040:0.0:1787764208.701067:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764208.701071:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787764208.701071:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000040:0.0:1787764208.701076:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764208.701077:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764208.701081:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787764208.701082:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764208.701085:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:1.0:1787764208.701087:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000020:00000040:0.0:1787764208.701088:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787764208.701091:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764208.701092:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00100000:1.0:1787764208.701094:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764208/real 1787764208]  req@ffff9751f876de00 x1874599832701824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787764208.701095:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787764208.701099:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787764208.701106:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764208.701111:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000001:1.0:1787764208.701113:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:0.0:1787764208.701114:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000001:1.0:1787764208.701116:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:0.0:1787764208.701117:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000010:1.0:1787764208.701120:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000020:00000040:0.0:1787764208.701121:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000400:00000200:1.0:1787764208.701124:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000100:00080000:0.0:1787764208.701126:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000010:1.0:1787764208.701129:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:0.0:1787764208.701130:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764208.701163:0:257575:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:0.0:1787764208.701164:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764208.701170:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:1.0:1787764208.701171:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876de00 x1874599832701824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787764208.701174:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764208.701180:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764208.701186:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000200:1.0:1787764208.701188:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876de00 x1874599832701824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:0.0:1787764208.701193:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764208.701197:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764208.701199:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764208.701202:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:0.0:1787764208.701203:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000100:00000001:1.0:1787764208.701204:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000040:0.0:1787764208.701208:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:1.0:1787764208.701208:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764208.701213:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876de00 x1874599832701824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787764208.701214:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764208.701216:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764208.701220:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764208.701222:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764208.701225:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764208.701227:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
02000000:00000010:0.0:1787764208.701230:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000040:1.0:1787764208.701234:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876de00 x1874599832701824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764208.701235:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764208.701245:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832702208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764208.701246:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764208.701252:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764208.701257:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764208.701259:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764208.701262:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764208.701263:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764208.701265:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787764208.701266:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764208.701270:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000040:0.0:1787764208.701273:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832702208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787764208.701277:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764208.701282:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764208.701289:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876de00 x1874599832701824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764208.701291:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764208.701299:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21933)
00000100:00100000:1.0:1787764208.701300:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832701824:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787764208.701307:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764208.701309:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764208.701311:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876de00 x1874599832701824/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.701315:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764208.701318:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764208.701320:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787764208.701322:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764208.701324:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764208.701325:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787764208.701326:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764208.701328:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014400.
02000000:00000001:1.0:1787764208.701331:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764208.701333:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764208.701334:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876de00.
00000100:00000001:1.0:1787764208.701337:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764208.701338:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764208.701339:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764208.701345:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764208.701346:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764208.701348:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764208.701349:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764208.701350:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764208.701353:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876da40 x1874599832701952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.701358:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764208.701359:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764208.701360:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764208.701361:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764208.701363:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764208.701365:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832701952:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764208.701367:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764208.701368:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764208.701369:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764208.701371:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764208.701373:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787764208.701375:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764208.701378:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764208.701380:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764208.701382:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764208.701385:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832701952, portal 10
00000100:00000001:1.0:1787764208.701386:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764208.701387:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049524800 : -115096660026816 : ffff9751f876da40)
00000100:00000040:1.0:1787764208.701392:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876da40 x1874599832701952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764233 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.701396:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764208.701398:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764208.701399:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764208.701401:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832701952, offset 0
00000400:00000200:1.0:1787764208.701403:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764208.701406:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764208.701410:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764208.701412:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764208.701414:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764208.701416:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764208.701417:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764208.701420:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58719:58719:256:4294967295] 192.168.203.110@tcp LPNI seq info [58719:58719:8:4294967295]
00000400:00000200:1.0:1787764208.701422:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764208.701427:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764208.701430:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764208.701433:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764208.701435:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764208.701437:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787764208.701441:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764208.701443:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787764208.701445:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764208.701447:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764208.701449:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764208.701451:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764208.701453:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764208.701455:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764208.701456:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764208.701458:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876da40 x1874599832701952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764233 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.701464:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764208.701466:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876da40 x1874599832701952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764233 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.701470:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764208.701471:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764208.701473:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764208.701476:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787764208.701478:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764208.701479:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316595744
00000100:00000001:1.0:1787764208.701482:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316596979
00000100:00000001:1.0:1787764208.701483:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316595744
00000100:00000001:1.0:1787764208.701484:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316596979
00000100:00000001:1.0:1787764208.701486:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764208.701487:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764208.701489:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764208.701491:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764208/real 1787764208]  req@ffff9751f876da40 x1874599832701952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764208.701495:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764208.701496:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764208.701497:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764208.701498:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764208.701499:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764208.701501:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764208.701502:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876da40 x1874599832701952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764208.701507:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876da40 x1874599832701952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.701512:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764208.701513:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764208.701514:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764208.701515:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764208.701519:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876da40 x1874599832701952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.701526:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764208.701531:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876da40 x1874599832701952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.701539:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764208.701541:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764208.701544:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764208.701545:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764208.701547:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764208.701559:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764208.701563:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764208.701565:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764208.701567:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764208.701570:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876da40 x1874599832701952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764208.701576:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832701952:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764208.701579:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764208.701581:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cb40 x1874599832702080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.701585:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764208.701586:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764208.701587:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764208.701589:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764208.701590:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764208.701592:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832702080:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764208.701594:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764208.701595:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764208.701596:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764208.701598:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764208.701600:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787764208.701602:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764208.701604:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764208.701606:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764208.701608:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764208.701610:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832702080, portal 10
00000100:00000001:1.0:1787764208.701611:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764208.701613:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049520960 : -115096660030656 : ffff9751f876cb40)
00000100:00000040:1.0:1787764208.701617:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cb40 x1874599832702080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764233 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.701621:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764208.701623:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764208.701624:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764208.701626:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832702080, offset 0
00000400:00000200:1.0:1787764208.701628:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764208.701631:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764208.701633:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764208.701635:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764208.701637:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764208.701639:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764208.701640:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764208.701643:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58720:58720:256:4294967295] 192.168.203.110@tcp LPNI seq info [58720:58720:8:4294967295]
00000400:00000200:1.0:1787764208.701645:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764208.701649:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764208.701652:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764208.701654:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764208.701656:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764208.701658:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787764208.701661:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764208.701663:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787764208.701665:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764208.701667:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764208.701669:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764208.701671:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764208.701673:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764208.701674:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764208.701676:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764208.701678:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cb40 x1874599832702080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764233 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.701683:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764208.701685:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cb40 x1874599832702080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764233 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.701690:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764208.701691:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764208.701693:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764208.701694:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787764208.701697:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764208.701698:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316595732
00000100:00000001:1.0:1787764208.701700:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316596979
00000100:00000001:1.0:1787764208.701701:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316595732
00000100:00000001:1.0:1787764208.701703:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316596979
00000100:00000001:1.0:1787764208.701704:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764208.701706:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764208.701708:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764208.701711:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764208/real 1787764208]  req@ffff9751f876cb40 x1874599832702080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764208.701720:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764208.701721:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764208.701723:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764208.701725:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764208.701727:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764208.701729:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764208.701733:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cb40 x1874599832702080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764208.701742:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cb40 x1874599832702080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.701750:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764208.701751:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764208.701753:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764208.701755:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764208.701759:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cb40 x1874599832702080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.701766:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764208.701768:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cb40 x1874599832702080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.701772:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764208.701774:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764208.701775:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764208.701776:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764208.701777:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764208.701778:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764208.701780:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764208.701782:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764208.701797:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764208.701801:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cb40 x1874599832702080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764208.701806:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832702080:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764208.701809:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764208.701811:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832702208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.701816:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764208.701817:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764208.701819:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764208.701820:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764208.701821:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764208.701823:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832702208:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764208.701825:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764208.701827:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764208.701828:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764208.701829:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764208.701832:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787764208.701833:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764208.701836:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764208.701838:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764208.701840:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764208.701842:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832702208, portal 4
00000100:00000001:1.0:1787764208.701844:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764208.701845:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
00000100:00000040:1.0:1787764208.701849:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832702208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764233 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.701853:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764208.701854:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764208.701856:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764208.701857:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832702208, offset 0
00000400:00000200:1.0:1787764208.701860:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764208.701862:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764208.701865:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764208.701867:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764208.701869:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764208.701870:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764208.701872:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764208.701874:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58721:58721:256:4294967295] 192.168.203.110@tcp LPNI seq info [58721:58721:8:4294967295]
00000400:00000200:1.0:1787764208.701878:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764208.701884:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764208.701889:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764208.701892:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764208.701896:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764208.701899:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:1.1:1787764208.701903:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764208.701906:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:1.0:1787764208.701910:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764208.701913:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764208.701916:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764208.701920:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764208.701923:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764208.701925:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764208.701927:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764208.701931:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876c000 x1874599832702208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764233 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.701937:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764208.701940:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832702208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764233 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.701944:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764208.701945:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764208.701948:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764208.701950:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787764208.701952:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764208.701953:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316595749
00000100:00000001:1.0:1787764208.701955:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316596979
00000100:00000001:1.0:1787764208.701957:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316595749
00000100:00000001:1.0:1787764208.701959:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316596979
00000100:00000001:1.0:1787764208.701960:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764208.701962:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764208.701964:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764208.701968:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764208/real 1787764208]  req@ffff9751f876c000 x1874599832702208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764208.701976:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764208.701977:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764208.701979:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764208.701982:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764208.701983:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764208.701986:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764208.701990:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832702208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764208.701999:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832702208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.702007:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764208.702009:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764208.702010:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764208.702011:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764208.702014:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832702208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.702019:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764208.702022:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832702208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.702026:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764208.702028:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764208.702029:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764208.702030:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764208.702031:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764208.702032:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764208.702033:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764208.702035:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764208.702036:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764208.702039:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832702208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764208.702043:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832702208:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764208.702046:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764208.702047:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764208.702049:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876da40 x1874599832701952/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.702054:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764208.702055:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764208.702057:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787764208.702058:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764208.702059:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764208.702061:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764208.702062:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764208.702063:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017000.
02000000:00000001:1.0:1787764208.702066:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764208.702067:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764208.702068:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876da40.
00000100:00000001:1.0:1787764208.702069:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764208.702070:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764208.702072:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764208.702074:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cb40 x1874599832702080/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.702078:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764208.702079:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764208.702080:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787764208.702081:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764208.702082:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764208.702083:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764208.702084:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764208.702085:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:1.0:1787764208.702086:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764208.702086:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764208.702087:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cb40.
00000100:00000001:1.0:1787764208.702089:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764208.702090:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764208.702091:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764208.702093:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832702208/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764233 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764208.702096:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764208.702098:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764208.702099:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787764208.702100:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764208.702101:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764208.702102:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787764208.702104:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764208.702104:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016800.
02000000:00000001:1.0:1787764208.702106:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764208.702107:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764208.702108:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:1.0:1787764208.702109:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764208.702110:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764208.702111:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764208.702113:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764208.702114:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764208.702115:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764208.702116:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764209.340244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:0.0:1787764209.340245:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764209.340251:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764209.340252:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764209.340257:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764209.340259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:0.0:1787764209.340261:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764209.340262:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764209.340267:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764209.340269:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:0.0:1787764209.340271:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764209.340272:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764209.340278:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764209.340281:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764209.340283:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764209.340307:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764209.340311:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764209.340314:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764209.724227:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764209.724235:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764209.724237:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764209.724244:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764209.724248:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764209.724252:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764209.724254:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764209.724257:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764209.724260:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764209.980239:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316596979, at 4316596992
00000100:00000010:3.0:1787764209.980258:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4a20.
00000100:00000040:2.0:1787764209.980291:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316596979, at 4316596992
00000100:00000010:2.0:1787764209.980300:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c43c0.
00010000:00000001:3.0:1787764210.364230:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764210.364235:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764210.364240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:0.0:1787764210.364241:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764210.364241:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764210.364247:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764210.364248:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764210.364252:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764210.364253:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764210.364258:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787764210.364261:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764210.364264:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764210.364267:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764210.364270:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764210.364273:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764210.364289:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764210.364294:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764210.364297:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764210.748246:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764210.748253:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764210.748255:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764210.748260:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764210.748264:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764210.748267:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764210.748270:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764210.748272:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764210.748274:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764211.388245:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787764211.388251:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764211.388252:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:0.0:1787764211.388253:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764211.388258:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764211.388259:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787764211.388264:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764211.388267:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:0.0:1787764211.388268:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764211.388273:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764211.388273:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764211.388277:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764211.388279:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:0.0:1787764211.388280:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764211.388281:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764211.388320:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764211.388324:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764211.388326:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764211.772210:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764211.772227:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764211.772229:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764211.772235:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764211.772239:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764211.772242:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764211.772245:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764211.772248:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764211.772251:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764212.412234:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764212.412237:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764212.412242:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764212.412243:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764212.412247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:0.0:1787764212.412248:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764212.412249:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764212.412254:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764212.412255:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764212.412260:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:0.0:1787764212.412261:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764212.412264:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:0.0:1787764212.412265:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764212.412267:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764212.412270:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764212.412271:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764212.412273:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764212.412276:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764212.412277:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764212.412280:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764212.412283:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764212.412320:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764212.412325:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764212.412329:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764212.796221:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764212.796228:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764212.796230:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764212.796234:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764212.796238:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764212.796242:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764212.796244:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764212.796246:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764212.796248:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764213.436239:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:0.0:1787764213.436240:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764213.436246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764213.436248:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764213.436251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:0.0:1787764213.436253:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764213.436254:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764213.436261:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:0.0:1787764213.436262:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764213.436263:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764213.436269:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764213.436274:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764213.436277:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764213.436281:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764213.436285:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764213.436305:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764213.436310:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764213.436315:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787764213.820223:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764213.820231:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787764213.820234:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764213.820237:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764213.820241:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787764213.820242:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764213.820247:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764213.820250:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764213.820251:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21928
00000100:00100000:2.0:1787764213.820255:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764213.820259:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764213.820262:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1787764213.820266:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764213.820266:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764213.820269:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1787764213.820277:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787764213.820282:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764213.820284:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764213.820290:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764213.820294:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764213.820298:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764213.820301:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764213.820304:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764213.820306:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764213.820308:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764213.820311:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764213.820313:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764213.820315:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787764213.820318:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764213.820319:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764213.820322:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764213.820323:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764213.820326:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787764213.820329:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764213.820330:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764213.820332:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764213.820335:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764213.820337:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764213.820339:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764213.820341:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764213.820348:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787764213.820351:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764213.820354:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764213.820355:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764213.820358:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764213.820360:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764213.820362:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764213.820369:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787764213.820375:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764213.820382:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832702336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764213.820390:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764213.820393:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764213.820397:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832702336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764213.820411:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764213.820413:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764213.820416:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764213.820417:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764213.820418:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764213.820420:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764213.820422:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21928
00000400:00000200:0.0:1787764213.820426:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764213.820429:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764213.820431:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787764213.820433:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1787764213.820434:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764213.820437:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787764213.820438:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:0.0:1787764213.820440:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764213.820441:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1787764213.820443:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764213.820446:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764213.820447:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787764213.820449:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:1.0:1787764213.820450:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832702336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787764213.820451:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787764213.820455:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787764213.820459:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787764213.820466:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764213.820469:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764213.820470:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787764213.820472:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764213.820474:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764213.820477:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:1.0:1787764213.820477:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:0.0:1787764213.820480:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787764213.820483:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764213.820484:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764213.820485:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764213.820487:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764213.820490:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764213.820491:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764213.820493:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764213.820497:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:1.0:1787764213.820498:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832702336:192.168.203.110@tcp:250:kworker.0
00000008:00100000:0.0:1787764213.820502:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:1.0:1787764213.820503:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:0.0:1787764213.820506:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764213.820508:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764213.820509:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:0.0:1787764213.820511:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
02000000:00000001:1.0:1787764213.820512:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787764213.820515:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787764213.820517:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764213.820519:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1787764213.820520:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764213.820523:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764213.820523:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764213.820525:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764213.820526:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cac00.
02000000:00000010:0.0:1787764213.820529:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
02000000:00000001:1.0:1787764213.820529:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764213.820535:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764213.820540:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00100000:0.0:1787764213.820544:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832702464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1787764213.820545:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000100:00000001:0.0:1787764213.820553:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:1.0:1787764213.820554:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000001:0.0:1787764213.820556:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764213.820560:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832702464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764213.820561:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832702336, portal 25
00000100:00000001:1.0:1787764213.820569:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:0.0:1787764213.820570:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764213.820571:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000100:00080000:0.0:1787764213.820576:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787764213.820580:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832702336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787764213.820582:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787764213.820586:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764213.820590:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764213.820591:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1787764213.820597:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:1.0:1787764213.820597:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764213.820602:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00080000:0.0:1787764213.820604:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21928
00000100:00000200:1.0:1787764213.820606:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832702336, offset 0
00000400:00000200:1.0:1787764213.820613:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764213.820619:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764213.820625:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764213.820629:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764213.820633:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764213.820636:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764213.820638:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764213.820641:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58722:58722:256:4294967295] 192.168.203.110@tcp LPNI seq info [58722:58722:8:4294967295]
00000400:00000200:1.0:1787764213.820644:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764213.820649:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764213.820657:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:1.0:1787764213.820660:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00080000:0.0:1787764213.820661:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764213.820668:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764213.820673:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.0:1787764213.820673:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787764213.820678:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000010:1.0:1787764213.820679:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000100:00000001:0.0:1787764213.820680:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764213.820684:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764213.820685:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787764213.820687:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:1.0:1787764213.820688:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:0.0:1787764213.820690:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787764213.820694:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:1.0:1787764213.820694:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000020:00000001:0.0:1787764213.820698:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:1.1:1787764213.820703:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787764213.820704:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764213.820709:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.1:1787764213.820710:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000100:00000040:0.0:1787764213.820712:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764213.820716:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764213.820720:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000800:00000100:1.0:1787764213.820720:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00080000:0.0:1787764213.820724:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000800:00000010:1.0:1787764213.820724:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000100:00000001:0.0:1787764213.820729:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764213.820730:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787764213.820731:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:0.0:1787764213.820733:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764213.820738:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764213.820738:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1787764213.820741:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:1.0:1787764213.820744:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000008:00100000:0.0:1787764213.820745:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000010:1.0:1787764213.820761:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000008:00000001:0.0:1787764213.820762:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764213.820766:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764213.820768:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000010:0.0:1787764213.820771:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000100:00000200:1.0:1787764213.820774:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e1c0 x1874599832702336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787764213.820775:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787764213.820780:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764213.820782:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764213.820786:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764213.820788:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764213.820791:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764213.820793:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000010:0.0:1787764213.820797:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000040:1.0:1787764213.820798:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832702336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764238 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764213.820803:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764213.820810:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764213.820812:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832702592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764213.820813:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787764213.820820:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764213.820820:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764213.820824:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764213.820828:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:0.0:1787764213.820829:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832702592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.820833:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316602099
00000100:00000001:0.0:1787764213.820840:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764213.820840:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4420.
00000100:00000040:0.0:1787764213.820846:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787764213.820849:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:1.0:1787764213.820849:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00080000:0.0:1787764213.820851:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764213.820855:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787764213.820855:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764213.820857:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764213.820857:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316602099
00000100:00000001:0.0:1787764213.820860:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000010:1.0:1787764213.820862:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c46c0.
00000100:00080000:0.0:1787764213.820864:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21928
00000100:00000001:1.0:1787764213.820868:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000400:00000200:0.0:1787764213.820872:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764213.820873:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:0.0:1787764213.820877:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:1.0:1787764213.820877:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764213.820880:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787764213.820881:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000040:0.0:1787764213.820886:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00100000:1.0:1787764213.820889:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764213/real 1787764213]  req@ffff9751f876e1c0 x1874599832702336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764213.820890:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764213.820895:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764213.820899:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764213.820904:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764213.820906:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787764213.820910:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764213.820913:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787764213.820917:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787764213.820920:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000001:1.0:1787764213.820921:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764213.820924:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:0.0:1787764213.820925:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000010:1.0:1787764213.820926:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000100:00000001:0.0:1787764213.820928:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764213.820931:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764213.820932:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764213.820934:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000400:00000200:1.0:1787764213.820935:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000100:00080000:0.0:1787764213.820936:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000010:1.0:1787764213.820938:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:0.0:1787764213.820940:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764213.820942:0:257575:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:0.0:1787764213.820943:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764213.820947:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:1.0:1787764213.820948:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832702336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787764213.820951:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764213.820956:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764213.820959:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000200:1.0:1787764213.820962:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832702336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:0.0:1787764213.820963:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764213.820967:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764213.820972:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000100:00000001:1.0:1787764213.820975:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000020:00000040:0.0:1787764213.820976:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:1.0:1787764213.820979:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787764213.820980:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764213.820982:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764213.820985:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764213.820986:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1787764213.820986:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:0.0:1787764213.820988:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764213.820990:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000010:0.0:1787764213.820993:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787764213.820996:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764213.820998:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832702336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787764213.821002:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832702720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764213.821028:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764213.821029:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764213.821032:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832702720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821038:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764213.821046:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832702336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764213.821048:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764213.821054:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21938)
00000100:00000001:1.0:1787764213.821056:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764213.821059:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764213.821061:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764213.821062:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764213.821063:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764213.821065:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764213.821067:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764213.821071:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764213.821072:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764213.821075:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832702336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764213.821081:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832702336:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787764213.821084:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764213.821086:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764213.821088:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832702336/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821092:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764213.821093:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764213.821095:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787764213.821097:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764213.821099:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764213.821100:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787764213.821101:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764213.821103:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014800.
02000000:00000001:1.0:1787764213.821105:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764213.821106:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764213.821107:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:1.0:1787764213.821111:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764213.821112:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764213.821113:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764213.821119:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764213.821120:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764213.821123:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764213.821124:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764213.821126:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764213.821130:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832702464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821161:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764213.821163:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764213.821165:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764213.821167:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764213.821169:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764213.821172:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832702464:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764213.821175:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764213.821177:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764213.821179:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764213.821181:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764213.821184:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787764213.821186:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764213.821190:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764213.821193:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764213.821197:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764213.821200:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832702464, portal 10
00000100:00000001:1.0:1787764213.821202:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764213.821203:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00000040:1.0:1787764213.821210:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832702464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821216:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764213.821218:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764213.821220:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764213.821223:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832702464, offset 0
00000400:00000200:1.0:1787764213.821226:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764213.821231:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764213.821234:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764213.821237:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764213.821239:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764213.821242:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764213.821244:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764213.821246:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58723:58723:256:4294967295] 192.168.203.110@tcp LPNI seq info [58723:58723:8:4294967295]
00000400:00000200:1.0:1787764213.821250:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764213.821256:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764213.821261:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764213.821264:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764213.821267:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764213.821270:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787764213.821275:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764213.821278:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787764213.821281:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764213.821283:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764213.821286:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764213.821290:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764213.821294:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764213.821295:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764213.821298:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764213.821302:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e580 x1874599832702464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821310:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764213.821313:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832702464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764238 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821320:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764213.821323:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764213.821326:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764213.821328:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787764213.821331:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764213.821333:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316600864
00000100:00000001:1.0:1787764213.821335:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316602099
00000100:00000001:1.0:1787764213.821337:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316600864
00000100:00000001:1.0:1787764213.821338:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316602099
00000100:00000001:1.0:1787764213.821340:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764213.821342:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764213.821344:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764213.821349:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764213/real 1787764213]  req@ffff9751f876e580 x1874599832702464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764213.821357:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764213.821359:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764213.821361:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764213.821364:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764213.821366:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764213.821368:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764213.821372:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832702464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764213.821381:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832702464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821388:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764213.821390:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764213.821392:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764213.821393:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764213.821397:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832702464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821401:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764213.821404:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832702464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821408:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764213.821410:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764213.821411:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764213.821411:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764213.821413:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764213.821422:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764213.821424:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764213.821426:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764213.821427:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764213.821430:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832702464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764213.821434:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832702464:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764213.821437:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764213.821440:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f0c0 x1874599832702592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821444:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764213.821445:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764213.821446:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764213.821447:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764213.821449:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764213.821450:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832702592:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764213.821453:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764213.821454:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764213.821455:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764213.821456:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764213.821459:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787764213.821460:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764213.821463:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764213.821464:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764213.821467:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764213.821469:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832702592, portal 10
00000100:00000001:1.0:1787764213.821471:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764213.821472:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049530560 : -115096660021056 : ffff9751f876f0c0)
00000100:00000040:1.0:1787764213.821476:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f0c0 x1874599832702592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821481:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764213.821482:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764213.821484:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764213.821485:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832702592, offset 0
00000400:00000200:1.0:1787764213.821488:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764213.821491:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764213.821493:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764213.821495:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764213.821497:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764213.821499:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764213.821501:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764213.821503:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58724:58724:256:4294967295] 192.168.203.110@tcp LPNI seq info [58724:58724:8:4294967295]
00000400:00000200:1.0:1787764213.821506:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764213.821510:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764213.821513:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764213.821515:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764213.821517:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764213.821519:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787764213.821522:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764213.821524:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787764213.821526:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764213.821528:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764213.821530:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764213.821532:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764213.821534:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764213.821535:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764213.821537:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764213.821539:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f0c0 x1874599832702592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821544:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764213.821546:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f0c0 x1874599832702592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764238 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821551:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764213.821552:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764213.821554:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764213.821556:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787764213.821557:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764213.821559:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316600852
00000100:00000001:1.0:1787764213.821560:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316602099
00000100:00000001:1.0:1787764213.821562:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316600852
00000100:00000001:1.0:1787764213.821564:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316602099
00000100:00000001:1.0:1787764213.821566:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764213.821567:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764213.821568:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764213.821571:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764213/real 1787764213]  req@ffff9751f876f0c0 x1874599832702592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764213.821575:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764213.821576:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764213.821577:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764213.821578:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764213.821579:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764213.821580:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764213.821582:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f0c0 x1874599832702592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764213.821587:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f0c0 x1874599832702592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821592:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764213.821593:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764213.821594:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764213.821594:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764213.821597:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f0c0 x1874599832702592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821601:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764213.821603:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f0c0 x1874599832702592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821607:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764213.821608:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764213.821609:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764213.821610:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764213.821612:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764213.821613:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764213.821615:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764213.821617:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764213.821619:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764213.821621:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f0c0 x1874599832702592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764213.821626:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832702592:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764213.821628:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764213.821630:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832702720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821634:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764213.821635:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764213.821636:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764213.821637:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764213.821638:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764213.821640:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832702720:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764213.821642:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764213.821644:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764213.821645:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764213.821646:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764213.821648:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787764213.821649:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764213.821651:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764213.821652:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764213.821655:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764213.821656:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832702720, portal 4
00000100:00000001:1.0:1787764213.821658:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764213.821659:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
00000100:00000040:1.0:1787764213.821662:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832702720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821667:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764213.821668:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764213.821669:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764213.821671:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832702720, offset 0
00000400:00000200:1.0:1787764213.821673:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764213.821675:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764213.821677:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764213.821679:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764213.821680:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764213.821682:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764213.821683:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764213.821685:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58725:58725:256:4294967295] 192.168.203.110@tcp LPNI seq info [58725:58725:8:4294967295]
00000400:00000200:1.0:1787764213.821688:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764213.821692:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764213.821694:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764213.821696:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764213.821698:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764213.821700:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:1.1:1787764213.821702:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764213.821704:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:1.0:1787764213.821706:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764213.821708:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764213.821709:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764213.821711:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764213.821713:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764213.821714:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764213.821716:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764213.821718:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876ed00 x1874599832702720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821723:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764213.821725:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832702720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764238 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821733:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764213.821734:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764213.821736:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764213.821738:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787764213.821740:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764213.821741:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316600869
00000100:00000001:1.0:1787764213.821743:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316602099
00000100:00000001:1.0:1787764213.821744:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316600869
00000100:00000001:1.0:1787764213.821745:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316602099
00000100:00000001:1.0:1787764213.821746:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764213.821748:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764213.821766:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764213.821769:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764213/real 1787764213]  req@ffff9751f876ed00 x1874599832702720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764213.821774:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764213.821775:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764213.821776:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764213.821778:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764213.821779:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764213.821780:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764213.821782:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832702720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764213.821788:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832702720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821791:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764213.821792:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764213.821793:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764213.821794:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764213.821796:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832702720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821801:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764213.821804:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832702720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821808:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764213.821810:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764213.821811:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764213.821811:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764213.821812:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764213.821813:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764213.821815:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764213.821817:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764213.821818:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764213.821820:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832702720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764213.821825:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832702720:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764213.821827:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764213.821828:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764213.821830:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832702464/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821834:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764213.821835:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764213.821837:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787764213.821839:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764213.821840:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764213.821841:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764213.821842:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764213.821844:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015c00.
02000000:00000001:1.0:1787764213.821846:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764213.821847:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764213.821848:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:1.0:1787764213.821850:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764213.821851:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764213.821852:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764213.821854:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f0c0 x1874599832702592/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821858:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764213.821859:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764213.821860:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787764213.821861:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764213.821862:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764213.821862:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764213.821864:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764213.821864:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016800.
02000000:00000001:1.0:1787764213.821866:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764213.821867:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764213.821868:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f0c0.
00000100:00000001:1.0:1787764213.821869:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764213.821870:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764213.821871:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764213.821872:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832702720/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764238 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764213.821876:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764213.821877:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764213.821878:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787764213.821879:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764213.821880:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764213.821880:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787764213.821881:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764213.821882:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:1.0:1787764213.821883:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764213.821884:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764213.821885:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:1.0:1787764213.821886:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764213.821887:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764213.821888:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764213.821891:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764213.821892:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764213.821893:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764213.821895:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764214.460247:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787764214.460254:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764214.460257:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764214.460261:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787764214.460266:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764214.460269:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764214.460273:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764214.460274:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:0.0:1787764214.460276:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764214.460282:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764214.460286:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764214.460292:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764214.460296:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764214.460302:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764214.460304:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764214.460358:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764214.460362:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764214.460365:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764214.844253:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764214.844260:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764214.844262:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764214.844267:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764214.844270:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764214.844274:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764214.844277:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764214.844279:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764214.844282:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764215.100263:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316602099, at 4316602112
00000100:00000010:3.0:1787764215.100283:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c46c0.
00000100:00000040:2.0:1787764215.100303:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316602099, at 4316602112
00000100:00000010:2.0:1787764215.100319:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4420.
00000100:00000001:0.0:1787764215.484208:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764215.484209:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:0.0:1787764215.484216:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764215.484217:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:0.0:1787764215.484219:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764215.484220:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764215.484223:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764215.484224:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764215.484227:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:0.0:1787764215.484229:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764215.484230:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764215.484232:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764215.484233:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764215.484237:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764215.484238:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764215.484239:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764215.484242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:0.0:1787764215.484243:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764215.484245:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764215.484250:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764215.484250:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764215.484286:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764215.484291:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764215.484295:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764215.868201:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764215.868208:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764215.868210:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764215.868215:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764215.868217:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764215.868220:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764215.868222:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764215.868224:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764215.868226:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764216.508241:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764216.508244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764216.508250:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764216.508252:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764216.508256:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764216.508257:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764216.508262:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764216.508263:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764216.508266:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:0.0:1787764216.508269:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764216.508270:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764216.508276:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764216.508281:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764216.508284:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764216.508287:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764216.508322:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764216.508326:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764216.508329:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764216.892236:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764216.892245:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764216.892248:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764216.892253:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764216.892258:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764216.892262:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764216.892266:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764216.892268:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764216.892271:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764217.532228:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764217.532232:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764217.532236:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764217.532237:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764217.532241:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764217.532244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764217.532248:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00100000:0.0:1787764217.532250:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764217.532250:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764217.532253:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764217.532255:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764217.532257:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:0.0:1787764217.532259:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764217.532260:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764217.532265:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764217.532265:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764217.532267:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764217.532269:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764217.532273:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764217.532275:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764217.532279:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764217.532318:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764217.532322:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764217.532324:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764217.916236:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764217.916245:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764217.916247:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764217.916254:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764217.916259:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764217.916263:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764217.916266:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764217.916268:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764217.916271:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764218.556246:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:0.0:1787764218.556253:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764218.556253:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764218.556259:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:0.0:1787764218.556261:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764218.556262:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764218.556267:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764218.556268:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764218.556270:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764218.556274:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787764218.556279:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764218.556282:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764218.556286:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764218.556289:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764218.556293:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764218.556303:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764218.556309:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764218.556313:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787764218.940249:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:2.0:1787764218.940250:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764218.940259:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1787764218.940263:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787764218.940263:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764218.940271:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:2.0:1787764218.940272:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:0.0:1787764218.940277:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764218.940278:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764218.940287:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787764218.940287:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764218.940297:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764218.940300:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764218.940302:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764218.940312:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21933
00000400:00000200:0.0:1787764218.940325:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764218.940334:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764218.940338:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764218.940343:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764218.940347:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764218.940351:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764218.940354:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764218.940359:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764218.940362:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764218.940365:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764218.940368:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764218.940370:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764218.940373:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787764218.940376:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764218.940378:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764218.940379:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764218.940381:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764218.940383:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787764218.940386:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764218.940387:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764218.940390:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764218.940393:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764218.940395:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764218.940397:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764218.940399:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764218.940410:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
00000020:00000040:0.0:1787764218.940413:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764218.940417:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764218.940418:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764218.940421:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764218.940424:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764218.940428:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764218.940436:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787764218.940450:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764218.940460:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832702848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764218.940468:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764218.940473:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764218.940476:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832702848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764218.940500:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764218.940502:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764218.940505:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764218.940507:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764218.940508:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764218.940511:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764218.940513:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21933
00000400:00000200:0.0:1787764218.940516:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764218.940518:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764218.940519:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764218.940521:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764218.940523:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764218.940525:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764218.940526:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764218.940529:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764218.940530:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787764218.940531:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764218.940533:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787764218.940535:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787764218.940537:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787764218.940542:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764218.940544:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764218.940545:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764218.940547:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764218.940549:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787764218.940551:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764218.940554:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787764218.940560:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764218.940564:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
02000000:00000001:0.0:1787764218.940565:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787764218.940572:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
02000000:00000001:0.0:1787764218.940575:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764218.940579:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764218.940583:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:1.0:1787764218.940590:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f840 x1874599832702848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1787764218.940591:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787764218.940597:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764218.940607:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764218.940615:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000020:00000040:0.0:1787764218.940620:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787764218.940629:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764218.940633:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764218.940634:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764218.940640:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1787764218.940641:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764218.940642:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764218.940645:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764218.940651:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
02000000:00000001:1.0:1787764218.940652:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1787764218.940655:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764218.940661:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832702976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:1.0:1787764218.940667:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764218.940677:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:1.0:1787764218.940679:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764218.940681:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764218.940691:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832702976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764218.940692:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832702848:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787764218.940711:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1787764218.940726:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764218.940729:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00080000:0.0:1787764218.940732:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:1.0:1787764218.940735:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764218.940738:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
02000000:00000001:1.0:1787764218.940741:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1787764218.940751:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
02000000:00000010:1.0:1787764218.940753:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
00000100:00080000:0.0:1787764218.940759:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
02000000:00000001:1.0:1787764218.940759:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764218.940767:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1787764218.940772:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:0.0:1787764218.940774:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21933
00000400:00000200:0.0:1787764218.940782:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764218.940784:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000200:0.0:1787764218.940788:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764218.940791:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764218.940795:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000010:1.0:1787764218.940795:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000001:0.0:1787764218.940798:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764218.940801:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764218.940806:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000200:1.0:1787764218.940806:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832702848, portal 25
00000100:00000001:1.0:1787764218.940821:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000020:00000001:0.0:1787764218.940822:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764218.940829:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000100:00000001:1.0:1787764218.940830:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049532480 : -115096660019136 : ffff9751f876f840)
00000020:00000001:0.0:1787764218.940841:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764218.940844:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000100:00000040:1.0:1787764218.940847:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f840 x1874599832702848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787764218.940848:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787764218.940860:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787764218.940872:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764218.940873:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1787764218.940875:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764218.940880:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764218.940882:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764218.940887:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00000040:1.0:1787764218.940890:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00080000:0.0:1787764218.940893:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764218.940896:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764218.940898:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:1.0:1787764218.940898:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
02000000:00000001:0.0:1787764218.940903:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764218.940907:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787764218.940907:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832702848, offset 0
00000100:00000001:0.0:1787764218.940909:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764218.940912:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787764218.940916:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764218.940917:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764218.940921:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787764218.940926:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000400:00000200:1.0:1787764218.940928:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
02000000:00000001:0.0:1787764218.940930:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764218.940936:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764218.940938:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764218.940939:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764218.940941:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764218.940944:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000001:0.0:1787764218.940949:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764218.940950:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00100000:0.0:1787764218.940962:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832703104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:1.0:1787764218.940964:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:0.0:1787764218.940981:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:1.0:1787764218.940982:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:0.0:1787764218.940986:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764218.940994:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832703104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787764218.940995:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:0.0:1787764218.941008:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764218.941009:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000040:0.0:1787764218.941012:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787764218.941018:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000040:0.0:1787764218.941019:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787764218.941027:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764218.941035:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000200:1.0:1787764218.941036:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58726:58726:256:4294967295] 192.168.203.110@tcp LPNI seq info [58726:58726:8:4294967295]
00000100:00000001:0.0:1787764218.941038:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764218.941040:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764218.941046:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764218.941052:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21933
00000400:00000200:1.0:1787764218.941057:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764218.941063:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764218.941069:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764218.941074:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764218.941080:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764218.941083:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764218.941091:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:1.0:1787764218.941094:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1787764218.941096:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764218.941104:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764218.941105:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787764218.941108:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764218.941113:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787764218.941115:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787764218.941119:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000800:00000200:1.0:1787764218.941124:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787764218.941125:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764218.941130:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764218.941168:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764218.941171:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000010:1.0:1787764218.941174:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000020:00000040:0.0:1787764218.941176:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787764218.941184:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764218.941194:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787764218.941195:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:0.0:1787764218.941201:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:1.0:1787764218.941205:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
02000000:00000001:0.0:1787764218.941208:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764218.941216:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.1:1787764218.941221:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787764218.941223:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764218.941228:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000800:00000200:1.1:1787764218.941230:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000008:00000001:0.0:1787764218.941234:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764218.941240:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:1.0:1787764218.941242:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764218.941247:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000100:00000010:0.0:1787764218.941249:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000400:00000200:1.0:1787764218.941255:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:0.0:1787764218.941256:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787764218.941266:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764218.941271:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764218.941275:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1787764218.941277:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764218.941280:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:1.0:1787764218.941283:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
02000000:00000001:0.0:1787764218.941284:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787764218.941291:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
02000000:00000010:0.0:1787764218.941294:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:1.0:1787764218.941299:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1787764218.941301:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:1.0:1787764218.941309:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f840 x1874599832702848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787764218.941313:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832703232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764218.941348:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1787764218.941349:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764218.941354:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764218.941359:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f840 x1874599832702848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764243 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764218.941365:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832703232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.941403:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764218.941406:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764218.941408:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:0.0:1787764218.941415:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21943)
00000100:00000001:1.0:1787764218.941418:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764218.941426:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764218.941429:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316607220
00000100:00000010:1.0:1787764218.941436:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c41e0.
00000100:00000001:1.0:1787764218.941443:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787764218.941451:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316607220
00000100:00000010:1.0:1787764218.941455:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4180.
00000100:00000001:1.0:1787764218.941458:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:1.0:1787764218.941462:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764218.941465:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764218.941467:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764218.941473:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764218/real 1787764218]  req@ffff9751f876f840 x1874599832702848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764218.941485:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764218.941486:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764218.941489:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764218.941492:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764218.941494:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764218.941496:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764218.941500:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f840 x1874599832702848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764218.941516:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f840 x1874599832702848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.941523:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764218.941525:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764218.941527:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764218.941529:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764218.941534:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f840 x1874599832702848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.941541:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764218.941545:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f840 x1874599832702848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.941553:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764218.941556:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764218.941557:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764218.941559:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764218.941562:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764218.941564:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764218.941568:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764218.941573:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764218.941576:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764218.941581:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f840 x1874599832702848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764218.941590:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832702848:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787764218.941594:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764218.941597:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764218.941600:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f840 x1874599832702848/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.941613:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764218.941616:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764218.941619:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787764218.941623:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764218.941626:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764218.941628:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787764218.941630:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764218.941633:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017000.
02000000:00000001:1.0:1787764218.941638:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764218.941640:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764218.941642:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f840.
00000100:00000001:1.0:1787764218.941645:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764218.941646:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764218.941648:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764218.941655:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764218.941657:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764218.941660:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764218.941662:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764218.941663:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764218.941668:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cf00 x1874599832702976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.941675:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764218.941676:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764218.941679:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764218.941681:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764218.941684:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764218.941687:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832702976:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764218.941691:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764218.941693:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764218.941695:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764218.941697:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764218.941701:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787764218.941703:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764218.941707:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764218.941710:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764218.941733:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764218.941738:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832702976, portal 10
00000100:00000001:1.0:1787764218.941740:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764218.941741:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049521920 : -115096660029696 : ffff9751f876cf00)
00000100:00000040:1.0:1787764218.941750:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cf00 x1874599832702976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.941757:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764218.941760:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764218.941763:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764218.941766:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832702976, offset 0
00000400:00000200:1.0:1787764218.941771:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764218.941777:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764218.941781:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764218.941785:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764218.941788:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764218.941791:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764218.941794:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764218.941798:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58727:58727:256:4294967295] 192.168.203.110@tcp LPNI seq info [58727:58727:8:4294967295]
00000400:00000200:1.0:1787764218.941803:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764218.941811:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764218.941816:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764218.941820:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764218.941824:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764218.941827:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787764218.941832:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764218.941835:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1787764218.941839:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764218.941841:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764218.941845:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764218.941848:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764218.941852:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764218.941854:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764218.941858:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764218.941862:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cf00 x1874599832702976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.941871:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764218.941874:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cf00 x1874599832702976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764243 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.941883:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764218.941885:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764218.941888:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764218.941893:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787764218.941896:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764218.941898:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316605984
00000100:00000001:1.0:1787764218.941902:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316607220
00000100:00000001:1.0:1787764218.941904:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316605984
00000100:00000001:1.0:1787764218.941906:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316607220
00000100:00000001:1.0:1787764218.941909:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764218.941911:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764218.941913:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764218.941918:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764218/real 1787764218]  req@ffff9751f876cf00 x1874599832702976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764218.941927:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764218.941928:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764218.941930:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764218.941933:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764218.941935:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764218.941939:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764218.941942:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cf00 x1874599832702976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764218.941951:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cf00 x1874599832702976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.941964:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764218.941966:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764218.941968:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764218.941969:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764218.941973:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cf00 x1874599832702976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.941980:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764218.941985:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cf00 x1874599832702976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.941992:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764218.941995:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764218.941996:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764218.941998:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764218.942000:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764218.942013:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764218.942017:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764218.942020:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764218.942023:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764218.942027:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cf00 x1874599832702976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764218.942036:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832702976:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764218.942041:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764218.942045:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e940 x1874599832703104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.942052:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764218.942054:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764218.942057:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764218.942059:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764218.942061:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764218.942064:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832703104:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764218.942068:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764218.942070:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764218.942072:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764218.942074:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764218.942078:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787764218.942080:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764218.942084:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764218.942087:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764218.942090:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764218.942094:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832703104, portal 10
00000100:00000001:1.0:1787764218.942096:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764218.942097:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049528640 : -115096660022976 : ffff9751f876e940)
00000100:00000040:1.0:1787764218.942104:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e940 x1874599832703104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.942111:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764218.942114:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764218.942117:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764218.942119:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832703104, offset 0
00000400:00000200:1.0:1787764218.942124:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764218.942128:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764218.942133:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764218.942160:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764218.942163:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764218.942166:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764218.942169:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764218.942172:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58728:58728:256:4294967295] 192.168.203.110@tcp LPNI seq info [58728:58728:8:4294967295]
00000400:00000200:1.0:1787764218.942177:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764218.942184:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764218.942190:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764218.942195:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764218.942199:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764218.942203:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787764218.942208:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764218.942211:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1787764218.942215:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764218.942218:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764218.942221:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764218.942225:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764218.942228:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764218.942230:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764218.942233:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764218.942238:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e940 x1874599832703104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.942246:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764218.942250:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e940 x1874599832703104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764243 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.942256:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764218.942258:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764218.942262:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764218.942265:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787764218.942269:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764218.942271:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316605973
00000100:00000001:1.0:1787764218.942274:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316607220
00000100:00000001:1.0:1787764218.942277:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316605973
00000100:00000001:1.0:1787764218.942279:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316607220
00000100:00000001:1.0:1787764218.942281:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764218.942283:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764218.942286:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764218.942290:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764218/real 1787764218]  req@ffff9751f876e940 x1874599832703104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764218.942299:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764218.942301:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764218.942303:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764218.942306:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764218.942307:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764218.942310:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764218.942313:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e940 x1874599832703104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764218.942323:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e940 x1874599832703104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.942330:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764218.942332:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764218.942334:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764218.942335:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764218.942339:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e940 x1874599832703104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.942347:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764218.942351:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e940 x1874599832703104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.942359:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764218.942361:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764218.942363:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764218.942364:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764218.942366:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764218.942368:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764218.942371:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764218.942375:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764218.942377:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764218.942383:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e940 x1874599832703104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764218.942392:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832703104:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764218.942396:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764218.942400:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832703232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.942408:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764218.942410:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764218.942412:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764218.942414:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764218.942417:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764218.942420:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832703232:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764218.942423:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764218.942426:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764218.942427:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764218.942429:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764218.942433:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787764218.942436:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764218.942439:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764218.942441:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764218.942445:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764218.942448:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832703232, portal 4
00000100:00000001:1.0:1787764218.942450:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764218.942451:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
00000100:00000040:1.0:1787764218.942457:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832703232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.942465:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764218.942467:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764218.942470:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764218.942473:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832703232, offset 0
00000400:00000200:1.0:1787764218.942478:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764218.942482:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764218.942485:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764218.942488:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764218.942491:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764218.942494:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764218.942497:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764218.942500:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58729:58729:256:4294967295] 192.168.203.110@tcp LPNI seq info [58729:58729:8:4294967295]
00000400:00000200:1.0:1787764218.942505:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764218.942511:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764218.942516:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764218.942520:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764218.942524:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764218.942527:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:1.1:1787764218.942531:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764218.942534:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:1.0:1787764218.942539:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764218.942541:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764218.942544:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764218.942547:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764218.942551:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764218.942552:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764218.942556:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764218.942560:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876ed00 x1874599832703232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.942569:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764218.942572:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832703232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764243 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.942580:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764218.942581:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764218.942585:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764218.942589:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787764218.942592:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764218.942594:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316605990
00000100:00000001:1.0:1787764218.942597:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316607220
00000100:00000001:1.0:1787764218.942600:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316605990
00000100:00000001:1.0:1787764218.942602:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316607220
00000100:00000001:1.0:1787764218.942604:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764218.942606:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764218.942608:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764218.942613:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764218/real 1787764218]  req@ffff9751f876ed00 x1874599832703232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764218.942621:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764218.942622:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764218.942624:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764218.942627:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764218.942629:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764218.942632:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764218.942635:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832703232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764218.942644:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832703232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.942651:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764218.942652:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764218.942655:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764218.942656:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764218.942660:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832703232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.942668:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764218.942672:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832703232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.942679:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764218.942683:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764218.942684:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764218.942685:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764218.942687:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764218.942689:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764218.942691:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764218.942695:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764218.942697:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764218.942701:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832703232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764218.942709:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832703232:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764218.942712:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764218.942731:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764218.942735:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cf00 x1874599832702976/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.942742:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764218.942743:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764218.942746:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787764218.942749:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764218.942750:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764218.942752:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764218.942754:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764218.942756:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014400.
02000000:00000001:1.0:1787764218.942758:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764218.942760:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764218.942761:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cf00.
00000100:00000001:1.0:1787764218.942765:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764218.942767:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764218.942769:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764218.942773:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e940 x1874599832703104/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.942787:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764218.942788:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764218.942790:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787764218.942793:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764218.942794:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764218.942795:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764218.942797:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764218.942799:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
02000000:00000001:1.0:1787764218.942802:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764218.942803:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764218.942805:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e940.
00000100:00000001:1.0:1787764218.942808:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764218.942810:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764218.942812:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764218.942816:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832703232/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764243 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764218.942824:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764218.942825:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764218.942827:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787764218.942830:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764218.942831:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764218.942832:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787764218.942834:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764218.942836:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:1.0:1787764218.942838:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764218.942840:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764218.942841:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:1.0:1787764218.942844:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764218.942846:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764218.942848:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764218.942852:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764218.942853:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764218.942856:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764218.942857:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764219.580220:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:0.0:1787764219.580227:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764219.580227:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:0.0:1787764219.580229:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764219.580234:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787764219.580237:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764219.580238:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764219.580242:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764219.580244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:0.0:1787764219.580245:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764219.580247:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764219.580248:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764219.580251:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:0.0:1787764219.580252:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764219.580253:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764219.580320:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764219.580325:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764219.580329:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764219.964273:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764219.964279:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764219.964281:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764219.964292:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764219.964296:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764219.964299:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764219.964302:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764219.964304:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764219.964308:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764220.220213:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316607220, at 4316607232
00000100:00000010:3.0:1787764220.220229:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4180.
00000100:00000040:2.0:1787764220.220268:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316607220, at 4316607232
00000100:00000010:2.0:1787764220.220278:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c41e0.
00000100:00000001:0.0:1787764220.604239:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764220.604244:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:0.0:1787764220.604246:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764220.604251:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764220.604251:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764220.604257:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:0.0:1787764220.604261:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764220.604261:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764220.604265:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764220.604266:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764220.604270:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764220.604271:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:0.0:1787764220.604272:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764220.604274:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764220.604276:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764220.604325:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764220.604330:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764220.604334:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764220.988262:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764220.988269:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764220.988272:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764220.988278:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764220.988281:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764220.988283:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764220.988285:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764220.988287:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764220.988290:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764221.628225:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764221.628227:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764221.628233:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764221.628239:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764221.628242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764221.628245:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764221.628246:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764221.628253:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764221.628255:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764221.628258:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764221.628260:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764221.628266:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764221.628269:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764221.628273:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764221.628277:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764221.628288:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764221.628293:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764221.628298:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764222.012266:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764222.012275:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764222.012278:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764222.012285:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764222.012290:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764222.012293:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764222.012297:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764222.012300:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764222.012304:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764222.652198:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764222.652212:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764222.652213:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764222.652218:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764222.652221:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764222.652222:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:1.0:1787764222.652225:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764222.652225:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764222.652231:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764222.652233:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764222.652236:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764222.652237:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:1.0:1787764222.652240:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764222.652241:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:1.0:1787764222.652243:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764222.652245:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764222.652247:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764222.652249:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764222.652252:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764222.652254:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764222.652256:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764222.652286:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764222.652291:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764222.652293:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764223.036231:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764223.036239:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764223.036242:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764223.036248:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764223.036252:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764223.036256:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764223.036259:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764223.036263:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764223.036266:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764223.676208:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764223.676215:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764223.676217:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764223.676222:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764223.676223:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764223.676226:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764223.676228:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764223.676230:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764223.676233:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764223.676236:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764223.676236:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764223.676238:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764223.676240:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764223.676246:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764223.676248:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764223.676297:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764223.676301:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764223.676305:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787764224.060293:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764224.060304:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787764224.060307:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764224.060311:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764224.060317:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764224.060324:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21938
00000400:00000200:0.0:1787764224.060333:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764224.060339:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000400:00000200:0.0:1787764224.060341:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:2.0:1787764224.060346:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1787764224.060347:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000001:2.0:1787764224.060352:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764224.060355:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00100000:2.0:1787764224.060358:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:0.0:1787764224.060362:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764224.060364:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:0.0:1787764224.060368:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764224.060368:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764224.060371:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764224.060374:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764224.060374:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764224.060379:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764224.060382:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764224.060386:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764224.060391:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764224.060395:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764224.060399:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764224.060415:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787764224.060419:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764224.060422:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764224.060424:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764224.060426:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764224.060430:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787764224.060433:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764224.060437:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764224.060440:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764224.060443:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764224.060446:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764224.060449:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764224.060452:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764224.060461:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787764224.060465:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764224.060469:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764224.060471:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764224.060474:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764224.060477:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764224.060481:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764224.060489:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:0.0:1787764224.060497:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764224.060506:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832703360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764224.060518:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764224.060523:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764224.060528:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832703360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764224.060555:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764224.060559:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764224.060564:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764224.060566:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764224.060568:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764224.060571:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764224.060574:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21938
00000400:00000200:0.0:1787764224.060579:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764224.060583:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764224.060586:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764224.060589:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764224.060593:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764224.060596:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764224.060596:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787764224.060602:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:2.0:1787764224.060603:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1787764224.060607:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:2.0:1787764224.060608:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1787764224.060612:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787764224.060617:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:2.0:1787764224.060618:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f0c0 x1874599832703360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787764224.060621:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787764224.060627:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787764224.060632:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:2.0:1787764224.060635:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764224.060639:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764224.060640:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764224.060644:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1787764224.060645:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:2.0:1787764224.060649:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1787764224.060650:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:2.0:1787764224.060655:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764224.060656:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764224.060663:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00100000:2.0:1787764224.060664:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832703360:192.168.203.110@tcp:250:kworker.0
00000100:00080000:0.0:1787764224.060670:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764224.060672:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764224.060691:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:0.0:1787764224.060693:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764224.060695:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764224.060698:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764224.060700:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1787764224.060705:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000010:2.0:1787764224.060708:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751c3910c00.
02000000:00000001:0.0:1787764224.060710:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764224.060714:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764224.060718:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:2.0:1787764224.060721:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:0.0:1787764224.060723:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764224.060729:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764224.060730:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000010:0.0:1787764224.060735:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000400:00000010:2.0:1787764224.060737:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da0ec8.
00000020:00000040:0.0:1787764224.060741:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787764224.060745:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000010:2.0:1787764224.060747:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec4c8.
02000000:00000001:0.0:1787764224.060748:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764224.060754:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764224.060755:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764224.060758:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764224.060758:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832703360, portal 25
00000100:00000001:2.0:1787764224.060764:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764224.060768:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049530560 : -115096660021056 : ffff9751f876f0c0)
02000000:00000010:0.0:1787764224.060769:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000100:00000001:0.0:1787764224.060776:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764224.060778:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f0c0 x1874599832703360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764249 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787764224.060786:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832703488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764224.060792:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764224.060800:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787764224.060803:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:2.0:1787764224.060806:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecdd0.
00000100:00000001:0.0:1787764224.060808:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:2.0:1787764224.060811:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832703360, offset 0
00000100:00000040:0.0:1787764224.060817:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832703488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764224.060824:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787764224.060836:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764224.060838:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:0.0:1787764224.060841:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764224.060848:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:2.0:1787764224.060849:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:0.0:1787764224.060853:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764224.060857:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:2.0:1787764224.060859:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764224.060865:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764224.060868:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764224.060873:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:0.0:1787764224.060875:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:2.0:1787764224.060880:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58730:58730:256:4294967295] 192.168.203.110@tcp LPNI seq info [58730:58730:8:4294967295]
00000100:00080000:0.0:1787764224.060883:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21938
00000400:00000200:2.0:1787764224.060891:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764224.060893:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764224.060899:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764224.060905:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1787764224.060909:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:0.0:1787764224.060913:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764224.060918:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:2.0:1787764224.060923:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787764224.060925:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764224.060930:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000010:2.0:1787764224.060931:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646fa00.
00000020:00000001:0.0:1787764224.060934:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764224.060938:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787764224.060941:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:2.0:1787764224.060943:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:0.0:1787764224.060945:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000200:2.1:1787764224.060951:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787764224.060958:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787764224.060961:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787764224.060966:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764224.060969:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:2.0:1787764224.060970:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764224.060973:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764224.060977:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:2.0:1787764224.060978:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:0.0:1787764224.060985:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00000001:2.0:1787764224.060985:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316612339
00000100:00080000:0.0:1787764224.060993:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000010:2.0:1787764224.060995:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7b40.
00000100:00000001:0.0:1787764224.060998:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764224.061002:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:2.0:1787764224.061004:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
02000000:00000001:0.0:1787764224.061006:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764224.061011:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764224.061014:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764224.061015:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316612339
00000008:00100000:0.0:1787764224.061017:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787764224.061022:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764224.061023:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750ecbb7cc0.
00000100:00000001:0.0:1787764224.061025:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764224.061030:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000010:0.0:1787764224.061033:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000100:00000001:2.0:1787764224.061035:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000040:0.0:1787764224.061038:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000001:2.0:1787764224.061041:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764224.061044:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:2.0:1787764224.061047:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764224.061050:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764224.061057:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
02000000:00000001:0.0:1787764224.061058:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764224.061061:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:0.0:1787764224.061063:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1787764224.061066:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:0.0:1787764224.061067:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787764224.061070:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
02000000:00000001:0.0:1787764224.061071:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764224.061074:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
02000000:00000010:0.0:1787764224.061080:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000040:2.0:1787764224.061082:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832703488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764224.061086:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764224.061097:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00100000:0.0:1787764224.061098:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832703616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764224.061101:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764224.061107:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764224.061110:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764224.061115:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:2.0:1787764224.061116:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764224.061122:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764224.061124:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832703488:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764224.061162:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764224.061166:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764224.061168:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764224.061170:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764224.061174:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751c3913000.
02000000:00000001:2.0:1787764224.061177:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764224.061181:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764224.061184:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da00b0.
00000400:00000010:2.0:1787764224.061189:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec5d8.
00000100:00000200:2.0:1787764224.061192:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832703488, portal 10
00000100:00000001:2.0:1787764224.061195:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764224.061196:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00000040:2.0:1787764224.061204:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832703488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764249 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764224.061213:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764224.061216:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764224.061219:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec6e8.
00000100:00000200:2.0:1787764224.061222:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832703488, offset 0
00000100:00000040:0.0:1787764224.061228:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832703616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764224.061241:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787764224.061245:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764224.061250:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787764224.061250:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764224.061257:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000040:0.0:1787764224.061259:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:2.0:1787764224.061263:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:0.0:1787764224.061264:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:2.0:1787764224.061269:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00080000:0.0:1787764224.061271:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787764224.061276:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764224.061278:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764224.061281:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764224.061285:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21938
00000400:00000200:0.0:1787764224.061290:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764224.061290:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764224.061298:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:0.0:1787764224.061300:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764224.061306:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:2.0:1787764224.061307:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58731:58731:255:4294967295] 192.168.203.110@tcp LPNI seq info [58731:58731:7:4294967295]
00000100:00000040:0.0:1787764224.061314:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764224.061317:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764224.061320:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764224.061327:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:2.0:1787764224.061332:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:0.0:1787764224.061333:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764224.061340:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:2.0:1787764224.061343:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000020:00000040:0.0:1787764224.061344:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787764224.061348:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000010:2.0:1787764224.061350:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f100.
00000020:00000040:0.0:1787764224.061351:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787764224.061356:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000800:00000200:2.0:1787764224.061358:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787764224.061361:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000800:00000200:2.1:1787764224.061367:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787764224.061370:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764224.061374:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764224.061376:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764224.061378:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764224.061382:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787764224.061386:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764224.061387:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764224.061389:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764224.061393:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
02000000:00000001:0.0:1787764224.061394:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764224.061400:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:2.0:1787764224.061401:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764224.061405:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316611104
02000000:00000001:0.0:1787764224.061407:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764224.061410:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316612339
00000100:00000001:0.0:1787764224.061412:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764224.061414:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316611104
00000008:00100000:0.0:1787764224.061417:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:2.0:1787764224.061419:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316612339
00000008:00000001:0.0:1787764224.061423:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764224.061425:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764224.061429:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764224.061430:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764224.061436:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764224.061438:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000100:00000001:2.0:1787764224.061439:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:0.0:1787764224.061444:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787764224.061448:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764224.061450:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764224.061451:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764224.061453:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:2.0:1787764224.061455:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
02000000:00000001:0.0:1787764224.061457:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:2.0:1787764224.061460:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:0.0:1787764224.061461:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764224.061464:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764224.061467:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
02000000:00000010:0.0:1787764224.061469:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000040:2.0:1787764224.061473:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832703616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764224.061475:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764224.061486:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00100000:0.0:1787764224.061488:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832703744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764224.061489:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764224.061495:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764224.061498:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764224.061503:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764224.061506:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00100000:2.0:1787764224.061510:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832703616:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787764224.061512:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764224.061519:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764224.061522:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:0.0:1787764224.061524:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832703744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764224.061525:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764224.061529:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764224.061535:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751c3910800.
02000000:00000001:2.0:1787764224.061539:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764224.061540:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764224.061545:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:0.0:1787764224.061548:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21948)
00000400:00000010:2.0:1787764224.061550:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da04d0.
00000400:00000010:2.0:1787764224.061556:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec198.
00000100:00000200:2.0:1787764224.061560:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832703616, portal 10
00000100:00000001:2.0:1787764224.061562:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764224.061564:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000100:00000040:2.0:1787764224.061572:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832703616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764249 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764224.061581:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764224.061583:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764224.061586:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecaa0.
00000100:00000200:2.0:1787764224.061589:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832703616, offset 0
00000400:00000200:2.0:1787764224.061594:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764224.061598:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764224.061602:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764224.061605:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764224.061608:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764224.061611:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764224.061614:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764224.061616:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58732:58732:254:4294967295] 192.168.203.110@tcp LPNI seq info [58732:58732:6:4294967295]
00000400:00000200:2.0:1787764224.061621:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764224.061627:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764224.061632:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764224.061636:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f800.
00000800:00000200:2.0:1787764224.061641:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764224.061645:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787764224.061648:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764224.061651:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:2.0:1787764224.061654:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764224.061656:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316611092
00000100:00000001:2.0:1787764224.061659:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316612339
00000100:00000001:2.0:1787764224.061662:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316611092
00000100:00000001:2.0:1787764224.061665:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316612339
00000100:00000001:2.0:1787764224.061667:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764224.061670:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764224.061674:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764224.061676:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764224.061694:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764224.061696:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1787764224.061699:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764224.061700:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764224.061701:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764224.061707:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832703744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764224.061716:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764224.061718:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764224.061721:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764224.061723:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764224.061726:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764224.061730:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832703744:192.168.203.110@tcp:8:kworker.0
00000100:00000001:2.0:1787764224.061734:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764224.061737:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764224.061738:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764224.061740:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:2.0:1787764224.061744:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff975202077800.
02000000:00000001:2.0:1787764224.061747:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764224.061750:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:2.0:1787764224.061754:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750d3da01b8.
00000400:00000010:2.0:1787764224.061758:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ec2a8.
00000100:00000200:2.0:1787764224.061761:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832703744, portal 4
00000100:00000001:2.0:1787764224.061763:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764224.061765:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
00000100:00000040:2.0:1787764224.061771:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832703744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764249 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764224.061778:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:2.0:1787764224.061781:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:2.0:1787764224.061784:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f49ecee0.
00000100:00000200:2.0:1787764224.061787:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832703744, offset 0
00000400:00000200:2.0:1787764224.061791:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764224.061796:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764224.061800:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:2.0:1787764224.061803:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1787764224.061806:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:2.0:1787764224.061808:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1787764224.061811:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:2.0:1787764224.061815:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58733:58733:253:4294967295] 192.168.203.110@tcp LPNI seq info [58733:58733:5:4294967295]
00000400:00000200:2.0:1787764224.061819:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764224.061825:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1787764224.061830:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:2.0:1787764224.061834:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f646f200.
00000800:00000200:2.0:1787764224.061839:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:2.1:1787764224.061843:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787764224.061846:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764224.061849:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:2.0:1787764224.061852:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764224.061854:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316611109
00000100:00000001:2.0:1787764224.061857:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316612339
00000100:00000001:2.0:1787764224.061859:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316611109
00000100:00000001:2.0:1787764224.061861:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316612339
00000100:00000001:2.0:1787764224.061865:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764224.061867:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764224.061870:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764224.061872:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764224.061875:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764224.061877:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1787764224.061879:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764224.061880:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764224.061882:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764224.061884:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764224.700289:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764224.700294:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764224.700300:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764224.700302:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764224.700307:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764224.700313:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764224.700318:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764224.700323:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764224.700324:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764224.700327:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764224.700329:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764224.700334:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:1.0:1787764224.700335:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764224.700339:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764224.700342:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764224.700401:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764224.700407:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764224.700412:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764225.084254:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764225.084268:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764225.084271:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764225.084276:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764225.084279:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764225.084282:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764225.084284:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764225.084286:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764225.084288:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764225.340268:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316612339, at 4316612352
00000100:00000040:2.0:1787764225.340295:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316612339, at 4316612352
00000100:00000010:3.0:1787764225.340297:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7cc0.
00000100:00000010:2.0:1787764225.340315:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750ecbb7b40.
00000100:00000001:1.0:1787764225.724189:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764225.724196:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764225.724198:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764225.724204:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:1.0:1787764225.724208:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764225.724212:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764225.724216:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764225.724218:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764225.724221:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764225.724242:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764225.724250:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764225.724254:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764225.724258:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764225.724263:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764225.724266:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764225.724268:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764225.724271:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764225.724277:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764225.724279:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764225.724282:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764225.724283:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764225.724332:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764225.724335:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764225.724338:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764226.108227:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764226.108235:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764226.108238:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764226.108244:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764226.108249:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764226.108253:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764226.108256:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764226.108259:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764226.108263:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764226.748286:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764226.748286:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764226.748296:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764226.748297:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764226.748300:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764226.748307:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764226.748309:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764226.748312:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764226.748314:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764226.748316:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764226.748321:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764226.748323:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764226.748324:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764226.748328:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764226.748329:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764226.748385:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764226.748389:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764226.748392:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764227.132227:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764227.132234:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764227.132236:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764227.132240:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764227.132243:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764227.132247:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764227.132250:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764227.132252:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764227.132255:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:2.0:1787764227.132560:0:257565:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.110:988
00000400:00000100:2.0:1787764227.132620:0:257565:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.110@tcp at host 192.168.203.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1787764227.132628:0:257565:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764227.132632:0:257565:0:(router.c:1791:lnet_notify()) 192.168.203.10@tcp notifying 192.168.203.110@tcp: down
00000400:00000200:2.0:1787764227.132636:0:257565:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764227.132640:0:257565:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1787764227.132644:0:257565:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:2.0:1787764227.132648:0:257565:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646fa00.
00000400:00000200:2.0:1787764227.132653:0:257565:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787764227.132670:0:257565:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787764227.132680:0:257565:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecdd0
00000400:00000010:2.0:1787764227.132683:0:257565:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecdd0.
00000100:00000001:2.0:1787764227.132687:0:257565:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764227.132696:0:257565:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f876f0c0 x1874599832703360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764249 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764227.132717:0:257565:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764227.132723:0:257565:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f0c0 x1874599832703360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764249 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764227.132732:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764227.132738:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764227.132739:0:257565:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764227.132744:0:257565:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764227.132745:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000800:00000100:2.0:1787764227.132756:0:257565:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:2.0:1787764227.132760:0:257565:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f100.
00000100:00100000:1.0:1787764227.132767:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764224/real 1787764227]  req@ffff9751f876f0c0 x1874599832703360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764227.132767:0:257565:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787764227.132778:0:257565:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787764227.132787:0:257565:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec6e8
00000400:00000010:2.0:1787764227.132791:0:257565:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec6e8.
00000100:00000001:2.0:1787764227.132795:0:257565:0:(events.c:34:request_out_callback()) Process entered
00000400:00000001:1.0:1787764227.132800:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000200:2.0:1787764227.132801:0:257565:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f876e580 x1874599832703488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764249 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764227.132805:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764227.132810:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da0ec8.
00000400:00000200:1.0:1787764227.132817:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec4c8
00000400:00000010:1.0:1787764227.132821:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec4c8.
00000100:00000001:2.0:1787764227.132827:0:257565:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787764227.132828:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:2.0:1787764227.132834:0:257565:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832703488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764249 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764227.132838:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f0c0 x1874599832703360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764227.132858:0:257565:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764227.132868:0:257565:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000200:1.0:1787764227.132872:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f0c0 x1874599832703360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:2.0:1787764227.132882:0:257565:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:2.0:1787764227.132888:0:257565:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f800.
00000100:00000001:1.0:1787764227.132892:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000400:00000200:2.0:1787764227.132894:0:257565:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00000001:1.0:1787764227.132897:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:2.0:1787764227.132901:0:257565:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000100:00000001:1.0:1787764227.132907:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000400:00000200:2.0:1787764227.132909:0:257565:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecaa0
00000100:00000001:1.0:1787764227.132912:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000400:00000010:2.0:1787764227.132913:0:257565:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecaa0.
00000100:00000001:2.0:1787764227.132919:0:257565:0:(events.c:34:request_out_callback()) Process entered
00000100:00100000:1.0:1787764227.132922:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f0c0 x1874599832703360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1787764227.132929:0:257565:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f876e1c0 x1874599832703616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764249 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764227.132941:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764227.132952:0:257565:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764227.132954:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f0c0 x1874599832703360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787764227.132960:0:257565:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832703616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764249 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764227.132973:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764227.132977:0:257565:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764227.132980:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764227.132982:0:257565:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764227.132985:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764227.132988:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000800:00000100:2.0:1787764227.132993:0:257565:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000100:00000001:1.0:1787764227.132994:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764227.132998:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000800:00000010:2.0:1787764227.132999:0:257565:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f646f200.
00000100:00080000:1.0:1787764227.133007:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000400:00000200:2.0:1787764227.133007:0:257565:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00080000:1.0:1787764227.133019:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000400:00000200:2.0:1787764227.133020:0:257565:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000100:00000001:1.0:1787764227.133026:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000400:00000200:2.0:1787764227.133028:0:257565:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ecee0
00000400:00000010:2.0:1787764227.133033:0:257565:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ecee0.
00000100:00000040:1.0:1787764227.133036:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f0c0 x1874599832703360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764227.133038:0:257565:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764227.133045:0:257565:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f876c000 x1874599832703744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764249 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764227.133057:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832703360:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764227.133066:0:257565:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787764227.133068:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764227.133073:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764227.133078:0:257565:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832703744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764249 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787764227.133079:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f0c0 x1874599832703360/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764227.133110:0:257565:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764227.133112:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
00000100:00000001:2.0:1787764227.133114:0:257565:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:1.0:1787764227.133118:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764227.133124:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751c3910c00.
02000000:00000001:1.0:1787764227.133131:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764227.133164:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764227.133167:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787764227.133170:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764227.133173:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016c00.
02000000:00000001:1.0:1787764227.133179:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764227.133182:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764227.133184:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f0c0.
00000100:00000001:1.0:1787764227.133188:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764227.133190:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764227.133192:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764227.133194:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764227.133195:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764227.133197:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764227.133202:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764224/real 1787764227]  req@ffff9751f876c000 x1874599832703744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764227.133210:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764227.133212:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764227.133215:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da01b8.
00000400:00000200:1.0:1787764227.133218:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec2a8
00000400:00000010:1.0:1787764227.133220:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec2a8.
00000100:00000001:1.0:1787764227.133223:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764227.133226:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832703744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764227.133233:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832703744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764227.133239:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764227.133241:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764227.133242:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764227.133244:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764227.133247:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832703744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764227.133253:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764227.133258:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832703744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764227.133265:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764227.133268:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764227.133269:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764227.133270:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764227.133272:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764227.133274:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764227.133276:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764227.133281:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764227.133283:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764227.133287:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832703744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764227.133295:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832703744:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764227.133300:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764227.133304:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764224/real 1787764227]  req@ffff9751f876e1c0 x1874599832703616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764227.133312:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764227.133313:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764227.133315:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da04d0.
00000400:00000200:1.0:1787764227.133317:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec198
00000400:00000010:1.0:1787764227.133319:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec198.
00000100:00000001:1.0:1787764227.133322:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764227.133325:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832703616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764227.133334:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832703616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764227.133340:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764227.133342:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764227.133344:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764227.133345:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764227.133349:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832703616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764227.133355:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764227.133359:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832703616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764227.133366:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764227.133368:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764227.133369:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764227.133371:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764227.133372:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764227.133374:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764227.133376:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764227.133380:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764227.133383:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764227.133387:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832703616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764227.133394:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832703616:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764227.133399:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764227.133402:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764224/real 1787764227]  req@ffff9751f876e580 x1874599832703488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764227.133411:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764227.133412:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764227.133414:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750d3da00b0.
00000400:00000200:1.0:1787764227.133417:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f49ec5d8
00000400:00000010:1.0:1787764227.133420:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f49ec5d8.
00000100:00000001:1.0:1787764227.133425:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764227.133429:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832703488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764227.133438:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832703488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764227.133446:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764227.133447:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764227.133449:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764227.133450:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764227.133454:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832703488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764227.133461:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764227.133465:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832703488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764227.133472:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764227.133474:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764227.133476:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764227.133477:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764227.133479:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764227.133496:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764227.133499:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764227.133504:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764227.133507:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764227.133513:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832703488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764227.133522:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832703488:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764227.133527:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764227.133530:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764227.133534:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832703744/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764227.133541:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764227.133543:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764227.133545:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff975202077800.
02000000:00000001:1.0:1787764227.133549:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764227.133550:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764227.133552:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787764227.133555:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764227.133557:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
02000000:00000001:1.0:1787764227.133560:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764227.133561:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764227.133563:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:1.0:1787764227.133568:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764227.133569:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764227.133572:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764227.133576:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832703616/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764227.133583:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764227.133585:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764227.133587:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751c3910800.
02000000:00000001:1.0:1787764227.133592:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764227.133593:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764227.133594:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764227.133596:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764227.133598:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:1.0:1787764227.133601:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764227.133602:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764227.133604:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:1.0:1787764227.133609:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764227.133611:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764227.133613:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764227.133618:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832703488/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764249 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764227.133632:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764227.133633:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764227.133635:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751c3913000.
02000000:00000001:1.0:1787764227.133639:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764227.133641:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764227.133642:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764227.133645:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764227.133647:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017800.
02000000:00000001:1.0:1787764227.133651:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764227.133653:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764227.133655:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:1.0:1787764227.133670:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764227.133671:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764227.133673:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764227.133681:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764227.133683:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764227.133686:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764227.133689:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764227.772204:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764227.772210:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764227.772212:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764227.772217:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764227.772222:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764227.772225:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764227.772227:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764227.772229:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764227.772231:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764227.772249:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764227.772257:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764227.772263:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764227.772268:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764227.772271:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764227.772275:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764227.772279:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764227.772282:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764227.772285:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764227.772288:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764227.772293:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764227.772294:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764227.772355:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764227.772360:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764227.772364:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764228.156256:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764228.156265:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764228.156267:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764228.156273:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764228.156277:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764228.156281:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764228.156285:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764228.156288:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764228.156293:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764228.796265:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764228.796271:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764228.796275:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764228.796277:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764228.796279:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764228.796287:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764228.796289:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764228.796292:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764228.796294:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764228.796297:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764228.796300:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764228.796300:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764228.796305:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764228.796308:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764228.796312:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764228.796338:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764228.796342:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764228.796346:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764229.180218:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764229.180229:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764229.180232:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764229.180239:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764229.180243:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764229.180247:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764229.180249:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764229.180252:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764229.180254:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764229.180278:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764229.180285:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787764229.180288:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764229.180291:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764229.180296:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764229.180303:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21943
00000400:00000200:0.0:1787764229.180311:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764229.180317:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764229.180321:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764229.180323:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764229.180326:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764229.180328:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764229.180330:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764229.180333:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764229.180335:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764229.180338:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764229.180341:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764229.180343:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764229.180345:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787764229.180347:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764229.180349:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764229.180350:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764229.180352:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764229.180354:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787764229.180357:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764229.180358:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764229.180361:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764229.180364:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764229.180366:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764229.180369:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764229.180371:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764229.180378:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000020:00000040:0.0:1787764229.180382:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764229.180388:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764229.180390:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764229.180393:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764229.180397:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764229.180402:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764229.180412:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787764229.180421:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764229.180435:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832703872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764229.180448:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764229.180454:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764229.180459:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832703872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764229.180486:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764229.180490:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764229.180494:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764229.180495:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764229.180497:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764229.180501:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764229.180504:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21943
00000400:00000200:0.0:1787764229.180509:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764229.180513:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764229.180516:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764229.180520:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764229.180523:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764229.180526:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764229.180529:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787764229.180529:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000001:0.0:1787764229.180536:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:1.0:1787764229.180537:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000040:0.0:1787764229.180538:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00000001:1.0:1787764229.180542:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1787764229.180543:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764229.180548:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787764229.180552:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787764229.180555:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cb40 x1874599832703872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787764229.180556:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787764229.180565:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764229.180569:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764229.180575:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764229.180576:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764229.180579:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787764229.180580:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787764229.180584:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:1.0:1787764229.180587:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:0.0:1787764229.180589:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:1.0:1787764229.180593:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764229.180594:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764229.180600:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764229.180601:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764229.180609:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764229.180614:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764229.180617:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:1.0:1787764229.180621:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832703872:192.168.203.110@tcp:250:kworker.0
00000008:00100000:0.0:1787764229.180623:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787764229.180632:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764229.180633:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1787764229.180635:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764229.180642:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:0.0:1787764229.180654:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
02000000:00000001:1.0:1787764229.180656:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787764229.180659:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764229.180662:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1787764229.180665:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764229.180669:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764229.180670:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
00000100:00000001:0.0:1787764229.180672:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764229.180674:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764229.180676:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764229.180681:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764229.180684:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787764229.180689:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764229.180694:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00100000:0.0:1787764229.180700:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832704000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:1.0:1787764229.180703:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000100:00000001:0.0:1787764229.180714:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000010:1.0:1787764229.180717:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000001:0.0:1787764229.180719:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764229.180726:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832704000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764229.180728:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832703872, portal 25
00000100:00000001:1.0:1787764229.180735:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764229.180739:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049520960 : -115096660030656 : ffff9751f876cb40)
00000100:00000001:0.0:1787764229.180743:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764229.180748:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:1.0:1787764229.180750:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cb40 x1874599832703872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764254 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787764229.180756:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787764229.180760:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764229.180763:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764229.180768:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1787764229.180769:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:1.0:1787764229.180775:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00080000:0.0:1787764229.180776:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21943
00000400:00000010:1.0:1787764229.180780:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000400:00000200:0.0:1787764229.180785:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000200:1.0:1787764229.180788:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832703872, offset 0
00000400:00000200:0.0:1787764229.180791:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764229.180796:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:1.0:1787764229.180802:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787764229.180803:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764229.180808:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764229.180811:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764229.180813:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764229.180817:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:1.0:1787764229.180818:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:0.0:1787764229.180821:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787764229.180825:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764229.180827:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000400:00000200:1.0:1787764229.180831:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000020:00000001:0.0:1787764229.180832:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787764229.180838:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:1.0:1787764229.180841:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:0.0:1787764229.180845:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764229.180851:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:1.0:1787764229.180851:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000040:0.0:1787764229.180857:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:1.0:1787764229.180861:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1787764229.180863:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764229.180870:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:1.0:1787764229.180870:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00080000:0.0:1787764229.180879:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:1.0:1787764229.180882:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58734:58734:256:4294967295] 192.168.203.110@tcp LPNI seq info [58734:58734:8:4294967295]
00000100:00000001:0.0:1787764229.180886:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764229.180891:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787764229.180892:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787764229.180895:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764229.180900:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764229.180903:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764229.180908:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:1.0:1787764229.180914:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000008:00000001:0.0:1787764229.180915:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764229.180920:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764229.180926:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000800:00000200:1.0:1787764229.180930:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000020:00000040:0.0:1787764229.180931:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787764229.180938:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764229.180939:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764229.180942:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000010:1.0:1787764229.180942:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
02000000:00000001:0.0:1787764229.180945:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764229.180949:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787764229.180955:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000010:0.0:1787764229.180956:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787764229.180963:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:1.0:1787764229.180964:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000100:00100000:0.0:1787764229.180972:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832704128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:1.1:1787764229.180978:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787764229.180988:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:1.1:1787764229.180989:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000100:00000001:0.0:1787764229.180991:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764229.180998:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832704128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000100:1.0:1787764229.181003:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787764229.181010:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:1.0:1787764229.181011:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000100:00000040:0.0:1787764229.181013:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787764229.181018:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:1.0:1787764229.181021:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:0.0:1787764229.181022:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764229.181027:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787764229.181029:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764229.181032:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:1.0:1787764229.181033:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1787764229.181035:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764229.181040:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21943
00000400:00000200:1.0:1787764229.181044:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000200:0.0:1787764229.181045:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764229.181050:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000400:00000200:0.0:1787764229.181053:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764229.181057:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:1.0:1787764229.181058:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:0.0:1787764229.181063:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764229.181067:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:1.0:1787764229.181069:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cb40 x1874599832703872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764254 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764229.181074:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764229.181079:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764229.181084:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764229.181089:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787764229.181092:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:1.0:1787764229.181092:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000040:0.0:1787764229.181095:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000040:1.0:1787764229.181100:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cb40 x1874599832703872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764254 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787764229.181101:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:1.0:1787764229.181110:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764229.181114:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:1.0:1787764229.181115:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:0.0:1787764229.181122:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764229.181125:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764229.181128:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764229.181128:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764229.181160:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764229.181164:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787764229.181167:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764229.181171:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764229.181173:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764229.181177:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764229.181179:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764229.181181:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764229.181184:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787764229.181186:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764229.181188:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764229.181195:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000020:00000040:0.0:1787764229.181199:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787764229.181203:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764229.181205:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764229.181207:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764229.181209:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764229.181213:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:1.0:1787764229.181213:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316617460
02000000:00000001:0.0:1787764229.181217:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764229.181224:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000010:1.0:1787764229.181225:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c48a0.
00000100:00000001:0.0:1787764229.181229:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764229.181238:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832704256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764229.181244:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:0.0:1787764229.181249:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764229.181253:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764229.181256:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316617460
00000100:00000040:0.0:1787764229.181261:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832704256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:1.0:1787764229.181265:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4d20.
00000100:00000001:1.0:1787764229.181272:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787764229.181276:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764229.181279:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1787764229.181285:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21953)
00000100:00000001:1.0:1787764229.181285:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764229.181303:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764229.181311:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764229/real 1787764229]  req@ffff9751f876cb40 x1874599832703872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764229.181324:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764229.181326:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764229.181328:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764229.181333:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764229.181335:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764229.181337:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764229.181342:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cb40 x1874599832703872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764229.181351:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cb40 x1874599832703872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.181359:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764229.181361:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764229.181364:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764229.181367:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764229.181372:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cb40 x1874599832703872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.181379:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764229.181385:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cb40 x1874599832703872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.181393:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764229.181396:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764229.181398:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764229.181399:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764229.181402:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764229.181405:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764229.181408:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764229.181413:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764229.181416:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764229.181420:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cb40 x1874599832703872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764229.181428:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832703872:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787764229.181435:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764229.181437:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764229.181441:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cb40 x1874599832703872/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.181456:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764229.181459:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764229.181462:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787764229.181465:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764229.181468:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764229.181470:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787764229.181472:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764229.181476:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014400.
02000000:00000001:1.0:1787764229.181481:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764229.181483:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764229.181485:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cb40.
00000100:00000001:1.0:1787764229.181490:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764229.181491:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764229.181494:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764229.181506:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764229.181507:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764229.181510:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764229.181512:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764229.181513:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764229.181518:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876da40 x1874599832704000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.181525:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764229.181527:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764229.181530:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764229.181532:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764229.181534:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764229.181538:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832704000:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764229.181542:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764229.181544:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764229.181547:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764229.181549:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764229.181553:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787764229.181555:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764229.181560:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764229.181564:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764229.181569:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764229.181573:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832704000, portal 10
00000100:00000001:1.0:1787764229.181575:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764229.181576:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049524800 : -115096660026816 : ffff9751f876da40)
00000100:00000040:1.0:1787764229.181585:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876da40 x1874599832704000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764254 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.181593:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764229.181595:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764229.181598:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764229.181601:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832704000, offset 0
00000400:00000200:1.0:1787764229.181605:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764229.181610:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764229.181613:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764229.181616:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764229.181619:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764229.181621:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764229.181623:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764229.181626:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58735:58735:256:4294967295] 192.168.203.110@tcp LPNI seq info [58735:58735:8:4294967295]
00000400:00000200:1.0:1787764229.181628:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764229.181634:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764229.181638:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764229.181640:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764229.181657:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764229.181661:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787764229.181666:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764229.181669:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1787764229.181674:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764229.181676:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764229.181680:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764229.181684:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764229.181687:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764229.181690:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764229.181694:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764229.181698:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876da40 x1874599832704000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764254 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.181707:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764229.181710:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876da40 x1874599832704000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764254 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.181716:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764229.181717:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764229.181721:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764229.181725:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787764229.181727:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764229.181729:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316616224
00000100:00000001:1.0:1787764229.181731:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316617460
00000100:00000001:1.0:1787764229.181733:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316616224
00000100:00000001:1.0:1787764229.181734:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316617460
00000100:00000001:1.0:1787764229.181736:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764229.181737:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764229.181739:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764229.181741:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764229/real 1787764229]  req@ffff9751f876da40 x1874599832704000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764229.181746:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764229.181747:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764229.181749:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764229.181750:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764229.181751:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764229.181753:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764229.181755:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876da40 x1874599832704000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764229.181761:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876da40 x1874599832704000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.181770:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764229.181771:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764229.181772:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764229.181773:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764229.181776:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876da40 x1874599832704000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.181780:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764229.181782:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876da40 x1874599832704000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.181786:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764229.181788:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764229.181789:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764229.181790:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764229.181791:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764229.181802:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764229.181804:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764229.181806:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764229.181807:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764229.181810:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876da40 x1874599832704000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764229.181816:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832704000:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764229.181819:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764229.181821:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876de00 x1874599832704128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.181825:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764229.181825:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764229.181827:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764229.181829:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764229.181830:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764229.181832:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832704128:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764229.181835:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764229.181837:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764229.181838:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764229.181839:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764229.181842:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787764229.181843:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764229.181845:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764229.181847:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764229.181849:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764229.181852:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832704128, portal 10
00000100:00000001:1.0:1787764229.181853:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764229.181854:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049525760 : -115096660025856 : ffff9751f876de00)
00000100:00000040:1.0:1787764229.181859:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876de00 x1874599832704128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764254 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.181863:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764229.181865:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764229.181866:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764229.181868:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832704128, offset 0
00000400:00000200:1.0:1787764229.181870:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764229.181873:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764229.181875:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764229.181877:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764229.181880:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764229.181881:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764229.181883:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764229.181885:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58736:58736:256:4294967295] 192.168.203.110@tcp LPNI seq info [58736:58736:8:4294967295]
00000400:00000200:1.0:1787764229.181888:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764229.181893:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764229.181902:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764229.181905:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764229.181907:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764229.181909:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787764229.181913:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764229.181915:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1787764229.181917:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764229.181919:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764229.181921:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764229.181923:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764229.181926:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764229.181927:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764229.181928:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764229.181931:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876de00 x1874599832704128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764254 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.181936:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764229.181938:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876de00 x1874599832704128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764254 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.181942:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764229.181943:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764229.181945:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764229.181947:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787764229.181949:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764229.181950:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316616212
00000100:00000001:1.0:1787764229.181952:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316617460
00000100:00000001:1.0:1787764229.181954:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316616212
00000100:00000001:1.0:1787764229.181955:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316617460
00000100:00000001:1.0:1787764229.181957:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764229.181958:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764229.181959:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764229.181962:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764229/real 1787764229]  req@ffff9751f876de00 x1874599832704128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764229.181968:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764229.181969:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764229.181970:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764229.181971:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764229.181972:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764229.181973:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764229.181975:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876de00 x1874599832704128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764229.181980:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876de00 x1874599832704128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.181985:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764229.181986:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764229.181987:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764229.181988:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764229.181990:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876de00 x1874599832704128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.181994:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764229.181997:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876de00 x1874599832704128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.182001:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764229.182002:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764229.182003:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764229.182004:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764229.182005:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764229.182006:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764229.182008:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764229.182010:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764229.182011:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764229.182014:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876de00 x1874599832704128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764229.182020:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832704128:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764229.182022:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764229.182025:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832704256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.182031:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764229.182032:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764229.182035:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764229.182037:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764229.182039:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764229.182042:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832704256:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764229.182046:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764229.182047:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764229.182049:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764229.182051:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764229.182055:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787764229.182057:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764229.182061:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764229.182063:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764229.182067:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764229.182070:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832704256, portal 4
00000100:00000001:1.0:1787764229.182072:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764229.182074:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00000040:1.0:1787764229.182081:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832704256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764254 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.182088:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764229.182090:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764229.182093:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764229.182096:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832704256, offset 0
00000400:00000200:1.0:1787764229.182101:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764229.182105:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764229.182108:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764229.182111:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764229.182114:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764229.182117:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764229.182120:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764229.182123:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58737:58737:256:4294967295] 192.168.203.110@tcp LPNI seq info [58737:58737:8:4294967295]
00000400:00000200:1.0:1787764229.182128:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764229.182165:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764229.182170:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764229.182174:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764229.182178:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764229.182181:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:1.1:1787764229.182186:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764229.182189:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:1.0:1787764229.182193:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764229.182195:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764229.182198:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764229.182202:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764229.182206:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764229.182208:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764229.182211:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764229.182214:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e580 x1874599832704256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764254 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.182223:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764229.182227:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832704256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764254 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.182234:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764229.182235:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764229.182239:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764229.182243:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787764229.182247:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764229.182250:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316616230
00000100:00000001:1.0:1787764229.182253:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316617460
00000100:00000001:1.0:1787764229.182255:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316616230
00000100:00000001:1.0:1787764229.182258:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316617460
00000100:00000001:1.0:1787764229.182260:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764229.182263:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764229.182265:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764229.182270:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764229/real 1787764229]  req@ffff9751f876e580 x1874599832704256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764229.182278:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764229.182280:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764229.182282:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764229.182285:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764229.182287:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764229.182289:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764229.182293:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832704256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764229.182302:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832704256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.182310:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764229.182311:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764229.182313:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764229.182315:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764229.182319:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832704256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.182331:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764229.182336:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832704256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.182341:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764229.182344:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764229.182345:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764229.182346:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764229.182348:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764229.182349:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764229.182351:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764229.182353:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764229.182354:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764229.182357:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832704256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764229.182362:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832704256:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764229.182364:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764229.182366:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764229.182368:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876da40 x1874599832704000/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.182373:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764229.182374:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764229.182376:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cb000.
02000000:00000001:1.0:1787764229.182378:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764229.182379:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764229.182380:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764229.182382:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764229.182384:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017000.
02000000:00000001:1.0:1787764229.182387:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764229.182388:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764229.182389:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876da40.
00000100:00000001:1.0:1787764229.182392:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764229.182393:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764229.182394:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764229.182396:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876de00 x1874599832704128/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.182405:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764229.182406:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764229.182407:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787764229.182408:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764229.182409:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764229.182409:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764229.182411:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764229.182411:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:1.0:1787764229.182413:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764229.182413:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764229.182414:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876de00.
00000100:00000001:1.0:1787764229.182417:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764229.182418:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764229.182419:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764229.182421:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832704256/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764254 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764229.182424:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764229.182425:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764229.182426:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787764229.182427:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764229.182428:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764229.182428:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787764229.182429:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764229.182430:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016800.
02000000:00000001:1.0:1787764229.182433:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764229.182433:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764229.182435:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:1.0:1787764229.182436:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764229.182438:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764229.182439:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764229.182444:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764229.182446:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764229.182448:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764229.182449:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764229.820224:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764229.820229:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764229.820231:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764229.820236:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764229.820236:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764229.820242:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764229.820247:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764229.820247:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764229.820252:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764229.820252:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764229.820257:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764229.820260:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764229.820263:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764229.820272:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764229.820276:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764229.820315:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764229.820321:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764229.820326:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764230.204257:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764230.204265:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764230.204268:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764230.204274:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764230.204278:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764230.204282:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764230.204285:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764230.204288:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764230.204292:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764230.460330:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316617460, at 4316617472
00000100:00000010:2.0:1787764230.460349:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c48a0.
00000100:00000040:3.0:1787764230.460420:0:304502:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316617460, at 4316617472
00000100:00000010:3.0:1787764230.460429:0:304502:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4d20.
00000100:00000001:2.0:1787764230.844256:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764230.844260:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764230.844264:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764230.844265:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764230.844267:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764230.844270:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764230.844271:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764230.844275:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764230.844276:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764230.844277:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764230.844281:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764230.844285:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764230.844287:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764230.844290:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764230.844294:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764230.844335:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764230.844341:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764230.844345:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764231.228274:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764231.228284:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764231.228287:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764231.228301:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764231.228306:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764231.228310:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764231.228314:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764231.228318:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764231.228321:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764231.868217:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764231.868220:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764231.868222:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764231.868229:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764231.868233:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764231.868239:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764231.868240:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764231.868243:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764231.868246:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764231.868248:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764231.868250:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764231.868252:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764231.868256:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764231.868259:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764231.868264:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764231.868309:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764231.868315:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764231.868320:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764232.252286:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764232.252297:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764232.252299:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764232.252305:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764232.252309:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764232.252314:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764232.252317:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764232.252319:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764232.252322:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764232.892272:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764232.892275:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764232.892279:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764232.892280:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764232.892283:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764232.892285:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764232.892286:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764232.892287:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764232.892291:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764232.892292:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764232.892294:0:304502:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764232.892295:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764232.892297:0:304502:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764232.892299:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764232.892301:0:304502:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764232.892302:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764232.892304:0:304502:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764232.892305:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764232.892307:0:304502:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764232.892309:0:304502:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764232.892311:0:304502:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764232.892362:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764232.892369:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764232.892375:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764233.276230:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764233.276237:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764233.276239:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764233.276243:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764233.276247:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764233.276249:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764233.276251:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764233.276253:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764233.276255:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764233.916249:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764233.916254:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764233.916259:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764233.916259:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764233.916263:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764233.916266:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764233.916269:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764233.916270:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764233.916277:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764233.916277:0:304502:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764233.916281:0:304502:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764233.916284:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764233.916288:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764233.916291:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764233.916295:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764233.916355:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764233.916360:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764233.916364:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764234.300284:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764234.300294:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764234.300297:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764234.300304:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764234.300309:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764234.300313:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764234.300317:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764234.300320:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764234.300323:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764234.300389:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764234.300399:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787764234.300403:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764234.300408:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764234.300414:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764234.300424:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21948
00000400:00000200:0.0:1787764234.300434:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764234.300443:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764234.300448:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764234.300455:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764234.300459:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764234.300463:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764234.300466:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764234.300471:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764234.300474:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764234.300477:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764234.300482:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764234.300485:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764234.300488:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787764234.300492:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764234.300494:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764234.300495:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764234.300497:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764234.300500:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787764234.300503:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764234.300505:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764234.300508:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764234.300512:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764234.300515:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764234.300518:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764234.300521:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764234.300529:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787764234.300533:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764234.300537:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764234.300539:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764234.300542:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764234.300546:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764234.300552:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764234.300562:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015c00.
00000100:00000001:0.0:1787764234.300574:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764234.300589:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832704384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764234.300602:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764234.300619:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764234.300626:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832704384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764234.300650:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764234.300654:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764234.300658:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764234.300660:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764234.300663:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764234.300667:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764234.300671:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21948
00000400:00000200:0.0:1787764234.300676:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764234.300680:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764234.300682:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764234.300684:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764234.300686:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764234.300688:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764234.300691:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764234.300693:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764234.300693:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787764234.300694:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764234.300696:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787764234.300697:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787764234.300699:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787764234.300703:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764234.300705:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764234.300707:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764234.300715:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764234.300718:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787764234.300720:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764234.300722:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764234.300724:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764234.300731:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764234.300732:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764234.300734:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764234.300739:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787764234.300742:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764234.300744:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764234.300748:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787764234.300750:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787764234.300752:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764234.300754:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764234.300756:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764234.300758:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764234.300761:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764234.300764:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787764234.300769:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764234.300775:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832704512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764234.300782:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764234.300784:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764234.300790:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832704512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764234.300795:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764234.300797:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764234.300800:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787764234.300802:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764234.300803:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764234.300807:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764234.300810:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21948
00000400:00000200:0.0:1787764234.300813:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764234.300819:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764234.300821:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764234.300824:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764234.300826:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787764234.300827:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764234.300833:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764234.300836:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764234.300837:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764234.300840:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764234.300843:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787764234.300845:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764234.300847:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787764234.300849:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787764234.300852:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787764234.300855:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764234.300858:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764234.300862:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764234.300865:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764234.300868:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787764234.300873:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764234.300876:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764234.300878:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764234.300882:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764234.300884:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
02000000:00000001:0.0:1787764234.300885:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764234.300890:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764234.300892:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787764234.300894:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764234.300896:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764234.300900:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000100:00000040:3.0:1787764234.300903:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832704384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787764234.300904:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787764234.300912:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764234.300914:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764234.300920:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764234.300922:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764234.300926:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764234.300936:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:3.0:1787764234.300943:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764234.300944:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764234.300946:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764234.300953:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832704640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1787764234.300953:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764234.300960:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764234.300969:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:3.0:1787764234.300970:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764234.300975:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764234.300980:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832704384:192.168.203.110@tcp:250:kworker.0
00000100:00000040:0.0:1787764234.300981:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832704640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.300995:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1787764234.301003:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764234.301006:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:3.0:1787764234.301006:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:0.0:1787764234.301013:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
02000000:00000001:3.0:1787764234.301013:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764234.301018:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:3.0:1787764234.301020:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:0.0:1787764234.301021:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787764234.301029:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764234.301030:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764234.301032:0:304484:0:(import.c:485:import_select_connection()) Process entered
02000000:00000010:3.0:1787764234.301034:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
00000100:00080000:0.0:1787764234.301037:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21948
02000000:00000001:3.0:1787764234.301043:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787764234.301044:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764234.301049:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764234.301052:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764234.301054:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:3.0:1787764234.301062:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764234.301063:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:3.0:1787764234.301073:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000100:00000040:0.0:1787764234.301074:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764234.301085:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764234.301088:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764234.301091:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000400:00000010:3.0:1787764234.301091:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000020:00000001:0.0:1787764234.301098:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764234.301101:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000200:3.0:1787764234.301103:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832704384, portal 25
00000020:00000001:0.0:1787764234.301104:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:3.0:1787764234.301111:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000020:00000001:0.0:1787764234.301112:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:3.0:1787764234.301116:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000100:00000040:0.0:1787764234.301122:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764234.301125:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764234.301132:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000040:3.0:1787764234.301158:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832704384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764259 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764234.301163:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764234.301169:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787764234.301182:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764234.301187:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764234.301188:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:0.0:1787764234.301194:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764234.301197:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1787764234.301199:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
02000000:00000001:0.0:1787764234.301201:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764234.301206:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764234.301208:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000400:00000010:3.0:1787764234.301209:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000008:00000001:0.0:1787764234.301211:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764234.301214:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764234.301216:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832704384, offset 0
00000100:00000010:0.0:1787764234.301219:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787764234.301222:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787764234.301225:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:3.0:1787764234.301228:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
02000000:00000001:0.0:1787764234.301229:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764234.301234:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764234.301235:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764234.301237:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764234.301241:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000400:00000200:3.0:1787764234.301241:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764234.301253:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764234.301259:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00100000:0.0:1787764234.301266:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832704768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787764234.301269:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764234.301281:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764234.301288:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764234.301300:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764234.301316:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58738:58738:256:4294967295] 192.168.203.110@tcp LPNI seq info [58738:58738:8:4294967295]
00000100:00000001:0.0:1787764234.301317:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764234.301325:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764234.301343:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:0.0:1787764234.301345:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832704768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764234.301365:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764234.301366:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:0.0:1787764234.301376:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21959)
00000800:00000200:3.0:1787764234.301378:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764234.301385:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764234.301391:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764234.301395:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1787764234.301401:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764234.301403:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:3.0:1787764234.301406:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764234.301408:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764234.301412:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764234.301415:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764234.301419:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764234.301420:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764234.301424:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764234.301427:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e1c0 x1874599832704384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764259 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.301435:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764234.301438:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832704384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764259 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.301443:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764234.301445:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764234.301449:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764234.301452:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764234.301454:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316622580
00000100:00000010:3.0:1787764234.301457:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d407e0.
00000100:00000001:3.0:1787764234.301462:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787764234.301467:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316622580
00000100:00000010:3.0:1787764234.301469:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40f00.
00000100:00000001:3.0:1787764234.301471:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787764234.301473:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764234.301475:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764234.301477:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764234.301480:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764234/real 1787764234]  req@ffff9751f876e1c0 x1874599832704384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764234.301491:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764234.301492:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764234.301494:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764234.301495:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764234.301496:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764234.301498:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764234.301500:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832704384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764234.301506:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832704384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.301510:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764234.301512:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764234.301513:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764234.301514:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764234.301517:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832704384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.301521:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764234.301524:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832704384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.301529:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764234.301531:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764234.301532:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764234.301533:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764234.301535:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764234.301536:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764234.301539:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764234.301542:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764234.301543:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764234.301546:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832704384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764234.301553:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832704384:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764234.301556:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764234.301559:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832704512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.301563:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764234.301564:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764234.301565:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764234.301567:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764234.301568:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764234.301570:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832704512:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764234.301572:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764234.301574:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764234.301575:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764234.301576:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764234.301579:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764234.301580:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764234.301582:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764234.301584:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764234.301586:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764234.301589:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832704512, portal 10
00000100:00000001:3.0:1787764234.301590:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764234.301591:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
00000100:00000040:3.0:1787764234.301596:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832704512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764259 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.301601:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764234.301603:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764234.301604:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764234.301605:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832704512, offset 0
00000400:00000200:3.0:1787764234.301626:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764234.301630:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764234.301632:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764234.301634:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764234.301636:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764234.301638:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764234.301640:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764234.301642:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58739:58739:256:4294967295] 192.168.203.110@tcp LPNI seq info [58739:58739:8:4294967295]
00000400:00000200:3.0:1787764234.301645:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764234.301649:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764234.301652:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764234.301655:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764234.301657:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764234.301659:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1787764234.301662:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764234.301664:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:3.0:1787764234.301667:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764234.301668:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764234.301670:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764234.301673:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764234.301675:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764234.301676:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764234.301678:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764234.301681:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876c000 x1874599832704512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764259 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.301687:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764234.301689:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832704512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764259 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.301693:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764234.301695:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764234.301697:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764234.301700:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764234.301702:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764234.301704:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316621344
00000100:00000001:3.0:1787764234.301706:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316622580
00000100:00000001:3.0:1787764234.301707:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316621344
00000100:00000001:3.0:1787764234.301709:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316622580
00000100:00000001:3.0:1787764234.301710:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764234.301711:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764234.301712:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764234.301715:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764234/real 1787764234]  req@ffff9751f876c000 x1874599832704512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764234.301720:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764234.301721:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764234.301722:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764234.301723:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764234.301725:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764234.301726:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764234.301728:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832704512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764234.301734:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832704512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.301737:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764234.301738:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764234.301739:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764234.301740:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764234.301742:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832704512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.301746:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764234.301749:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832704512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.301753:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764234.301755:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764234.301756:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764234.301757:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764234.301759:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764234.301768:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764234.301770:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764234.301772:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764234.301774:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764234.301777:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832704512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764234.301781:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832704512:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764234.301784:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764234.301785:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764234.301787:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832704384/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.301791:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764234.301793:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764234.301795:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764234.301797:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764234.301799:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764234.301800:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764234.301802:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764234.301804:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015c00.
02000000:00000001:3.0:1787764234.301806:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764234.301808:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764234.301809:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:3.0:1787764234.301813:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764234.301814:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764234.301815:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764234.301817:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832704512/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.301826:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764234.301827:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764234.301828:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764234.301829:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764234.301830:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764234.301831:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764234.301832:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764234.301833:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014800.
02000000:00000001:3.0:1787764234.301834:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764234.301835:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764234.301835:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:3.0:1787764234.301837:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764234.301838:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764234.301839:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764234.301845:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764234.301846:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764234.301848:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764234.301849:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764234.301850:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764234.301852:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f0c0 x1874599832704640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.301857:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764234.301858:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764234.301859:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764234.301860:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764234.301861:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764234.301863:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832704640:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764234.301865:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764234.301867:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764234.301868:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764234.301870:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764234.301872:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764234.301873:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764234.301875:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764234.301877:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764234.301879:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764234.301881:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832704640, portal 10
00000100:00000001:3.0:1787764234.301883:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764234.301884:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049530560 : -115096660021056 : ffff9751f876f0c0)
00000100:00000040:3.0:1787764234.301888:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f0c0 x1874599832704640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764259 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.301893:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764234.301894:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764234.301896:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764234.301897:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832704640, offset 0
00000400:00000200:3.0:1787764234.301900:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764234.301902:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764234.301904:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764234.301906:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764234.301908:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764234.301910:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764234.301911:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764234.301913:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58740:58740:256:4294967295] 192.168.203.110@tcp LPNI seq info [58740:58740:8:4294967295]
00000400:00000200:3.0:1787764234.301916:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764234.301920:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764234.301923:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764234.301926:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764234.301928:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764234.301930:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1787764234.301933:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764234.301935:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:3.0:1787764234.301938:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764234.301939:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764234.301941:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764234.301943:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764234.301945:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764234.301947:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764234.301948:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764234.301950:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f0c0 x1874599832704640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764259 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.301955:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764234.301957:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f0c0 x1874599832704640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764259 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.301961:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764234.301962:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764234.301964:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764234.301966:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764234.301968:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764234.301969:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316621332
00000100:00000001:3.0:1787764234.301971:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316622580
00000100:00000001:3.0:1787764234.301972:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316621332
00000100:00000001:3.0:1787764234.301973:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316622580
00000100:00000001:3.0:1787764234.301975:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764234.301976:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764234.301977:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764234.301979:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764234/real 1787764234]  req@ffff9751f876f0c0 x1874599832704640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764234.301984:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764234.301984:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764234.301986:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764234.301987:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764234.301988:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764234.301990:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764234.301992:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f0c0 x1874599832704640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764234.301997:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f0c0 x1874599832704640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.302004:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764234.302006:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764234.302008:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764234.302009:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764234.302013:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f0c0 x1874599832704640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.302020:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764234.302024:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f0c0 x1874599832704640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.302031:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764234.302034:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764234.302036:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764234.302037:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764234.302039:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764234.302041:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764234.302044:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764234.302047:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764234.302050:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764234.302054:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f0c0 x1874599832704640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764234.302060:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832704640:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764234.302063:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764234.302065:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832704768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.302070:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764234.302071:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764234.302073:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764234.302074:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764234.302076:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764234.302078:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832704768:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764234.302080:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764234.302082:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764234.302083:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764234.302084:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764234.302086:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764234.302088:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764234.302090:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764234.302092:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764234.302095:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764234.302097:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832704768, portal 4
00000100:00000001:3.0:1787764234.302099:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764234.302100:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
00000100:00000040:3.0:1787764234.302104:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832704768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764259 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.302108:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764234.302109:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764234.302111:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764234.302112:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832704768, offset 0
00000400:00000200:3.0:1787764234.302115:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764234.302117:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764234.302119:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764234.302121:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764234.302123:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764234.302125:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764234.302127:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764234.302129:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58741:58741:256:4294967295] 192.168.203.110@tcp LPNI seq info [58741:58741:8:4294967295]
00000400:00000200:3.0:1787764234.302161:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764234.302169:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764234.302175:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764234.302179:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764234.302183:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764234.302185:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000200:3.1:1787764234.302187:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764234.302189:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 52 secs later)
00000800:00000100:3.0:1787764234.302192:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764234.302193:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764234.302195:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764234.302198:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764234.302200:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764234.302201:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764234.302203:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764234.302206:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876ed00 x1874599832704768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764259 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.302211:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764234.302213:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832704768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764259 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.302218:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764234.302219:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764234.302221:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764234.302223:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764234.302226:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764234.302227:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316621350
00000100:00000001:3.0:1787764234.302229:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316622580
00000100:00000001:3.0:1787764234.302231:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316621350
00000100:00000001:3.0:1787764234.302232:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316622580
00000100:00000001:3.0:1787764234.302234:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764234.302236:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764234.302238:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764234.302251:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764234/real 1787764234]  req@ffff9751f876ed00 x1874599832704768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764234.302264:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764234.302266:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764234.302268:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764234.302271:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764234.302272:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764234.302275:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764234.302278:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832704768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764234.302287:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832704768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.302292:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764234.302293:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764234.302295:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764234.302296:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764234.302299:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832704768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.302303:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764234.302305:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832704768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.302310:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764234.302311:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764234.302312:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764234.302313:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764234.302315:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764234.302316:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764234.302317:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764234.302319:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764234.302321:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764234.302324:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832704768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764234.302329:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832704768:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764234.302331:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764234.302332:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764234.302334:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f0c0 x1874599832704640/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.302338:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764234.302339:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764234.302341:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764234.302342:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764234.302343:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764234.302344:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764234.302346:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764234.302347:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016800.
02000000:00000001:3.0:1787764234.302349:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764234.302350:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764234.302351:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f0c0.
00000100:00000001:3.0:1787764234.302353:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764234.302354:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764234.302355:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764234.302357:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832704768/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764259 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764234.302361:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764234.302362:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764234.302363:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764234.302364:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764234.302365:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764234.302365:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764234.302366:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764234.302367:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:3.0:1787764234.302368:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764234.302370:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764234.302370:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:3.0:1787764234.302372:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764234.302373:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764234.302374:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764234.302377:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764234.302378:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764234.302379:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764234.302381:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764234.940247:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764234.940256:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764234.940259:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764234.940265:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764234.940267:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764234.940270:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764234.940271:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764234.940275:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764234.940277:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764234.940280:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764234.940281:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764234.940283:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764234.940286:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764234.940287:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764234.940292:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764234.940336:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764234.940340:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764234.940343:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764235.324239:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764235.324247:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764235.324249:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764235.324257:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764235.324262:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764235.324265:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764235.324268:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764235.324271:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764235.324274:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764235.580305:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316622580, at 4316622592
00000100:00000010:3.0:1787764235.580320:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40f00.
00000100:00000040:2.0:1787764235.580343:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316622580, at 4316622592
00000100:00000010:2.0:1787764235.580352:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d407e0.
00000100:00000001:2.0:1787764235.964249:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764235.964260:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764235.964262:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764235.964268:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764235.964270:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764235.964273:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764235.964277:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764235.964278:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764235.964282:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764235.964283:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764235.964289:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764235.964289:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764235.964292:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764235.964296:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764235.964302:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764235.964305:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764235.964308:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764235.964314:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764235.964317:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764235.964322:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764235.964323:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764235.964368:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764235.964374:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764235.964378:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764236.348213:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764236.348238:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764236.348242:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764236.348249:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764236.348255:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764236.348260:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764236.348264:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764236.348267:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764236.348272:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764236.988276:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764236.988283:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764236.988284:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764236.988286:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764236.988289:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764236.988292:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764236.988295:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764236.988296:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764236.988298:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764236.988300:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764236.988302:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764236.988305:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764236.988305:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764236.988308:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764236.988312:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764236.988363:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764236.988369:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764236.988374:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764237.372291:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764237.372299:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764237.372302:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764237.372308:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764237.372312:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764237.372316:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764237.372319:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764237.372321:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764237.372324:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764238.012301:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764238.012308:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764238.012311:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764238.012315:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764238.012321:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764238.012323:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764238.012325:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764238.012327:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00100000:2.0:1787764238.012332:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764238.012333:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764238.012337:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764238.012338:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764238.012342:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764238.012343:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764238.012347:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764238.012348:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764238.012350:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764238.012352:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764238.012354:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764238.012358:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764238.012361:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764238.012435:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764238.012440:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764238.012443:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764238.396266:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764238.396278:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764238.396281:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764238.396285:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764238.396290:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764238.396294:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764238.396297:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764238.396300:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764238.396303:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764239.036226:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764239.036234:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764239.036236:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764239.036242:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764239.036246:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764239.036250:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764239.036252:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764239.036255:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764239.036258:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764239.036263:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764239.036267:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764239.036272:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764239.036273:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764239.036276:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764239.036277:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764239.036314:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764239.036319:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764239.036322:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764239.420258:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764239.420267:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764239.420268:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764239.420274:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:1.0:1787764239.420275:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764239.420285:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787764239.420293:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:2.0:1787764239.420294:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:1.0:1787764239.420300:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764239.420301:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764239.420310:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:2.0:1787764239.420310:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764239.420320:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764239.420324:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764239.420326:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21954
00000400:00000200:1.0:1787764239.420337:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764239.420345:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764239.420347:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764239.420350:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764239.420353:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764239.420355:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764239.420357:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764239.420360:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764239.420361:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787764239.420364:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764239.420367:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787764239.420369:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787764239.420370:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787764239.420372:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764239.420374:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764239.420376:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764239.420377:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764239.420380:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787764239.420383:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764239.420385:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764239.420387:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764239.420390:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764239.420392:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764239.420394:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764239.420396:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764239.420407:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930780.
00000020:00000040:1.0:1787764239.420409:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787764239.420414:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764239.420416:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764239.420420:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764239.420423:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764239.420428:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764239.420440:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9400.
00000100:00000001:1.0:1787764239.420449:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764239.420462:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930780 x1874599832704896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764239.420472:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764239.420476:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764239.420480:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3977950] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930780 x1874599832704896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764239.420504:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764239.420506:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764239.420509:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787764239.420510:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764239.420512:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764239.420514:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764239.420517:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21954
00000400:00000200:1.0:1787764239.420520:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764239.420522:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764239.420524:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764239.420527:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764239.420528:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764239.420530:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764239.420532:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764239.420534:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764239.420535:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787764239.420536:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764239.420538:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787764239.420539:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787764239.420541:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787764239.420543:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764239.420544:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764239.420546:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764239.420548:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764239.420550:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787764239.420552:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764239.420554:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764239.420556:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764239.420557:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
02000000:00000001:1.0:1787764239.420561:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764239.420563:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
02000000:00000001:1.0:1787764239.420566:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764239.420567:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1787764239.420569:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764239.420585:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000040:3.0:1787764239.420587:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930780 x1874599832704896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:1.0:1787764239.420588:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764239.420594:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764239.420600:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932580.
00000100:00000001:3.0:1787764239.420602:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:1.0:1787764239.420604:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000100:00000001:3.0:1787764239.420605:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764239.420609:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764239.420611:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764239.420613:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764239.420615:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764239.420616:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764239.420620:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764239.420620:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764239.420624:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764239.420626:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832704896:192.168.203.110@tcp:250:kworker.0
02000000:00000010:1.0:1787764239.420629:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9c00.
00000100:00000001:3.0:1787764239.420631:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1787764239.420634:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764239.420636:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764239.420639:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764239.420642:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932580 x1874599832705024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1787764239.420644:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1787764239.420651:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000010:3.0:1787764239.420651:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
00000100:00000001:1.0:1787764239.420654:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764239.420656:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764239.420659:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d026924] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932580 x1874599832705024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764239.420667:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764239.420673:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764239.420673:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000100:00080000:1.0:1787764239.420675:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764239.420679:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787764239.420680:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000010:3.0:1787764239.420682:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00080000:1.0:1787764239.420683:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764239.420688:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764239.420691:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21954
00000100:00000200:3.0:1787764239.420693:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832704896, portal 25
00000400:00000200:1.0:1787764239.420696:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764239.420698:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764239.420701:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497723264 : -115097211828352 : ffff9751d7930780)
00000400:00000200:1.0:1787764239.420702:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764239.420707:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764239.420712:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000040:3.0:1787764239.420713:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930780 x1874599832704896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764264 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764239.420715:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764239.420720:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764239.420724:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764239.420728:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764239.420731:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000100:00000001:3.0:1787764239.420732:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000020:00000001:1.0:1787764239.420733:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764239.420736:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000100:00000040:3.0:1787764239.420740:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000020:00000001:1.0:1787764239.420741:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787764239.420745:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000010:3.0:1787764239.420745:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764239.420750:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832704896, offset 0
00000100:00000040:1.0:1787764239.420752:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764239.420755:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764239.420756:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764239.420758:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764239.420759:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:3.0:1787764239.420763:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00080000:1.0:1787764239.420765:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764239.420768:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764239.420770:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764239.420772:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764239.420773:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764239.420777:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764239.420778:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764239.420781:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787764239.420783:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764239.420785:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764239.420788:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933840.
00000400:00000200:3.0:1787764239.420788:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000020:00000040:1.0:1787764239.420793:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764239.420796:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:3.0:1787764239.420797:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:1.0:1787764239.420798:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764239.420800:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764239.420801:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1787764239.420803:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
02000000:00000001:1.0:1787764239.420804:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764239.420809:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb000.
00000400:00000200:3.0:1787764239.420809:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1787764239.420814:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764239.420815:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00100000:1.0:1787764239.420823:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933840 x1874599832705152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787764239.420823:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58742:58742:256:4294967295] 192.168.203.110@tcp LPNI seq info [58742:58742:8:4294967295]
00000400:00000200:3.0:1787764239.420836:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764239.420841:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764239.420843:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764239.420848:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000083dd6d32] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933840 x1874599832705152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764239.420854:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:1.0:1787764239.420868:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787764239.420868:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000040:1.0:1787764239.420874:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000010:3.0:1787764239.420877:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000100:00000040:1.0:1787764239.420878:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787764239.420883:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764239.420886:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000800:00000200:3.0:1787764239.420887:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787764239.420888:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764239.420891:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:3.0:1787764239.420893:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000100:00000001:1.0:1787764239.420894:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764239.420899:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21954
00000800:00000200:3.1:1787764239.420904:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000400:00000200:1.0:1787764239.420905:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764239.420911:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000800:00000200:3.1:1787764239.420911:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000100:00080000:1.0:1787764239.420914:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000100:3.0:1787764239.420919:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000040:1.0:1787764239.420920:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764239.420924:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000010:3.0:1787764239.420924:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000100:00000040:1.0:1787764239.420927:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764239.420930:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764239.420932:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764239.420933:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000400:00000200:3.0:1787764239.420934:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:1.0:1787764239.420935:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764239.420938:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787764239.420941:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000200:3.0:1787764239.420942:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:1.0:1787764239.420945:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000200:3.0:1787764239.420949:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000100:00000040:1.0:1787764239.420950:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000010:3.0:1787764239.420954:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:1.0:1787764239.420955:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764239.420959:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764239.420961:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764239.420962:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000020:00000040:1.0:1787764239.420963:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:1.0:1787764239.420969:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000200:3.0:1787764239.420970:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930780 x1874599832704896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764264 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764239.420972:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764239.420977:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764239.420980:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764239.420984:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764239.420988:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764239.420992:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1787764239.420995:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764239.420997:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764239.420997:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000010:1.0:1787764239.421002:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932940.
00000100:00000040:3.0:1787764239.421003:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930780 x1874599832704896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764264 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787764239.421006:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787764239.421011:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764239.421014:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764239.421018:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1787764239.421018:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764239.421019:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1787764239.421022:0:257575:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:1.0:1787764239.421023:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764239.421028:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8000.
00000100:00000001:3.0:1787764239.421030:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764239.421031:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764239.421039:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00100000:1.0:1787764239.421040:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932940 x1874599832705280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764239.421043:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316627699
00000100:00000001:1.0:1787764239.421051:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000010:3.0:1787764239.421051:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40f00.
00000100:00000001:1.0:1787764239.421053:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764239.421057:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000af90eaa] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932940 x1874599832705280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421060:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787764239.421064:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764239.421069:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21964)
00000100:00000001:3.0:1787764239.421072:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316627699
00000100:00000010:3.0:1787764239.421076:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d403c0.
00000100:00000001:3.0:1787764239.421079:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787764239.421082:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764239.421084:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764239.421086:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764239.421091:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764239/real 1787764239]  req@ffff9751d7930780 x1874599832704896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764239.421106:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764239.421107:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764239.421109:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764239.421111:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764239.421112:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764239.421114:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764239.421117:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930780 x1874599832704896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764239.421123:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930780 x1874599832704896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421127:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764239.421129:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764239.421173:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764239.421175:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764239.421179:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930780 x1874599832704896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421184:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764239.421187:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930780 x1874599832704896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421192:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764239.421195:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764239.421196:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764239.421196:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764239.421199:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764239.421200:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764239.421202:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764239.421206:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764239.421207:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764239.421210:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930780 x1874599832704896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764239.421216:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832704896:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764239.421219:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764239.421221:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764239.421223:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930780 x1874599832704896/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421227:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764239.421229:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764239.421232:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764239.421234:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764239.421236:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764239.421237:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764239.421239:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764239.421242:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9400.
02000000:00000001:3.0:1787764239.421245:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764239.421246:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764239.421248:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930780.
00000100:00000001:3.0:1787764239.421250:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764239.421251:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764239.421253:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764239.421259:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764239.421260:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764239.421262:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764239.421263:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764239.421264:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764239.421266:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932580 x1874599832705024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421272:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764239.421273:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764239.421274:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764239.421275:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764239.421277:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764239.421279:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832705024:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764239.421280:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764239.421282:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764239.421283:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764239.421284:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764239.421287:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764239.421288:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764239.421291:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764239.421293:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764239.421295:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764239.421298:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832705024, portal 10
00000100:00000001:3.0:1787764239.421299:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764239.421300:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497730944 : -115097211820672 : ffff9751d7932580)
00000100:00000040:3.0:1787764239.421306:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932580 x1874599832705024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764264 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421311:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764239.421312:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764239.421314:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764239.421315:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832705024, offset 0
00000400:00000200:3.0:1787764239.421318:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764239.421320:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764239.421322:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764239.421325:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764239.421326:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764239.421328:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764239.421330:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764239.421332:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58743:58743:256:4294967295] 192.168.203.110@tcp LPNI seq info [58743:58743:8:4294967295]
00000400:00000200:3.0:1787764239.421335:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764239.421340:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764239.421343:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764239.421345:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764239.421348:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764239.421356:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787764239.421360:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764239.421361:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:3.0:1787764239.421364:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764239.421365:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764239.421368:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764239.421370:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764239.421372:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764239.421373:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764239.421375:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764239.421378:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932580 x1874599832705024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764264 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421383:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764239.421385:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932580 x1874599832705024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764264 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421394:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764239.421395:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764239.421398:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764239.421401:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764239.421403:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764239.421405:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316626464
00000100:00000001:3.0:1787764239.421407:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316627699
00000100:00000001:3.0:1787764239.421409:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316626464
00000100:00000001:3.0:1787764239.421410:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316627699
00000100:00000001:3.0:1787764239.421412:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764239.421413:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764239.421414:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764239.421416:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764239/real 1787764239]  req@ffff9751d7932580 x1874599832705024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764239.421421:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764239.421422:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764239.421423:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764239.421425:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764239.421426:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764239.421427:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764239.421429:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932580 x1874599832705024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764239.421434:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932580 x1874599832705024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421438:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764239.421439:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764239.421440:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764239.421442:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764239.421444:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932580 x1874599832705024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421453:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764239.421456:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932580 x1874599832705024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421461:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764239.421462:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764239.421463:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764239.421464:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764239.421465:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764239.421476:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764239.421478:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764239.421480:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764239.421482:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764239.421485:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932580 x1874599832705024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764239.421489:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832705024:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764239.421492:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764239.421494:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933840 x1874599832705152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421498:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764239.421498:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764239.421499:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764239.421501:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764239.421502:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764239.421504:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832705152:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764239.421506:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764239.421508:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764239.421509:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764239.421510:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764239.421513:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764239.421514:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764239.421516:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764239.421518:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764239.421520:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764239.421522:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832705152, portal 10
00000100:00000001:3.0:1787764239.421523:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764239.421524:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497735744 : -115097211815872 : ffff9751d7933840)
00000100:00000040:3.0:1787764239.421529:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933840 x1874599832705152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764264 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421533:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764239.421535:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764239.421536:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764239.421538:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832705152, offset 0
00000400:00000200:3.0:1787764239.421540:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764239.421542:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764239.421544:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764239.421546:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764239.421548:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764239.421549:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764239.421551:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764239.421553:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58744:58744:256:4294967295] 192.168.203.110@tcp LPNI seq info [58744:58744:8:4294967295]
00000400:00000200:3.0:1787764239.421556:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764239.421560:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764239.421562:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764239.421564:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764239.421567:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764239.421569:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787764239.421583:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764239.421585:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:3.0:1787764239.421587:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764239.421589:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764239.421591:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764239.421594:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764239.421596:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764239.421597:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764239.421599:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764239.421601:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933840 x1874599832705152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764264 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421607:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764239.421609:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933840 x1874599832705152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764264 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421613:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764239.421614:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764239.421616:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764239.421618:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764239.421620:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764239.421622:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316626452
00000100:00000001:3.0:1787764239.421624:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316627699
00000100:00000001:3.0:1787764239.421625:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316626452
00000100:00000001:3.0:1787764239.421627:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316627699
00000100:00000001:3.0:1787764239.421628:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764239.421629:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764239.421631:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764239.421633:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764239/real 1787764239]  req@ffff9751d7933840 x1874599832705152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764239.421637:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764239.421638:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764239.421639:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764239.421641:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764239.421642:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764239.421644:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764239.421646:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933840 x1874599832705152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764239.421651:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933840 x1874599832705152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421655:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764239.421656:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764239.421657:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764239.421658:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764239.421660:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933840 x1874599832705152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421664:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764239.421666:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933840 x1874599832705152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421670:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764239.421672:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764239.421672:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764239.421673:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764239.421674:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764239.421675:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764239.421677:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764239.421679:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764239.421681:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764239.421683:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933840 x1874599832705152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764239.421688:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832705152:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764239.421690:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764239.421693:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932940 x1874599832705280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421698:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764239.421699:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764239.421700:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764239.421701:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764239.421703:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764239.421704:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832705280:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764239.421706:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764239.421707:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764239.421708:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764239.421710:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764239.421712:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764239.421713:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764239.421716:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764239.421717:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764239.421719:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764239.421722:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832705280, portal 4
00000100:00000001:3.0:1787764239.421723:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764239.421724:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497731904 : -115097211819712 : ffff9751d7932940)
00000100:00000040:3.0:1787764239.421729:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932940 x1874599832705280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764264 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421733:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764239.421735:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764239.421736:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764239.421738:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832705280, offset 0
00000400:00000200:3.0:1787764239.421740:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764239.421742:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764239.421744:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764239.421746:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764239.421747:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764239.421749:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764239.421751:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764239.421753:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58745:58745:256:4294967295] 192.168.203.110@tcp LPNI seq info [58745:58745:8:4294967295]
00000400:00000200:3.0:1787764239.421756:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764239.421760:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764239.421763:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764239.421765:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764239.421767:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764239.421769:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000200:3.1:1787764239.421772:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764239.421774:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 47 secs later)
00000800:00000100:3.0:1787764239.421776:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764239.421778:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764239.421780:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764239.421782:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764239.421784:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764239.421785:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764239.421786:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764239.421789:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932940 x1874599832705280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764264 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421794:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764239.421796:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932940 x1874599832705280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764264 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421800:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764239.421801:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764239.421802:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764239.421804:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764239.421806:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764239.421807:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316626469
00000100:00000001:3.0:1787764239.421809:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316627699
00000100:00000001:3.0:1787764239.421810:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316626469
00000100:00000001:3.0:1787764239.421812:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316627699
00000100:00000001:3.0:1787764239.421813:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764239.421815:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764239.421816:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764239.421819:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764239/real 1787764239]  req@ffff9751d7932940 x1874599832705280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764239.421824:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764239.421825:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764239.421826:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764239.421827:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764239.421828:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764239.421829:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764239.421831:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932940 x1874599832705280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764239.421837:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932940 x1874599832705280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421841:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764239.421842:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764239.421843:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764239.421843:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764239.421845:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932940 x1874599832705280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421849:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764239.421852:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932940 x1874599832705280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421856:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764239.421857:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764239.421858:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764239.421859:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764239.421860:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764239.421861:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764239.421862:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764239.421864:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764239.421866:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764239.421869:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932940 x1874599832705280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764239.421874:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832705280:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764239.421876:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764239.421877:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764239.421879:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932580 x1874599832705024/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421882:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764239.421883:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764239.421885:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764239.421886:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764239.421887:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764239.421888:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764239.421889:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764239.421890:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:3.0:1787764239.421891:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764239.421892:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764239.421893:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932580.
00000100:00000001:3.0:1787764239.421895:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764239.421896:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764239.421897:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764239.421899:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933840 x1874599832705152/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421903:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764239.421904:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764239.421905:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764239.421907:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764239.421907:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764239.421908:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764239.421909:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764239.421910:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb000.
02000000:00000001:3.0:1787764239.421912:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764239.421913:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764239.421914:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933840.
00000100:00000001:3.0:1787764239.421917:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764239.421918:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764239.421919:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764239.421921:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932940 x1874599832705280/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764264 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764239.421924:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764239.421925:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764239.421926:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764239.421927:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764239.421928:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764239.421928:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764239.421929:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764239.421930:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8000.
02000000:00000001:3.0:1787764239.421932:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764239.421932:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764239.421933:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932940.
00000100:00000001:3.0:1787764239.421934:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764239.421935:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764239.421936:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764239.421938:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764239.421939:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764239.421940:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764239.421941:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000008:00000020:0.0:1787764239.932311:0:304600:0:(osc_request.c:747:osc_update_next_shrink()) next time 23159 to shrink grant
00000008:00000020:0.0:1787764239.932318:0:304600:0:(osc_request.c:747:osc_update_next_shrink()) next time 23159 to shrink grant
00000008:00000020:0.0:1787764239.932319:0:304600:0:(osc_request.c:747:osc_update_next_shrink()) next time 23159 to shrink grant
00000100:00000001:2.0:1787764240.060262:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764240.060268:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764240.060271:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764240.060274:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764240.060274:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764240.060282:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764240.060286:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764240.060287:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764240.060290:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764240.060291:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764240.060294:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764240.060296:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764240.060299:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764240.060307:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764240.060310:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764240.060393:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764240.060399:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764240.060402:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764240.444290:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764240.444307:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764240.444309:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764240.444315:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764240.444319:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764240.444323:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764240.444325:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764240.444328:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764240.444331:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764240.700305:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316627699, at 4316627712
00000100:00000010:3.0:1787764240.700319:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d403c0.
00000100:00000040:2.0:1787764240.700354:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316627699, at 4316627712
00000100:00000010:2.0:1787764240.700363:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40f00.
00000100:00000001:2.0:1787764241.084207:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764241.084216:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764241.084218:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764241.084223:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764241.084225:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764241.084228:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764241.084230:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764241.084232:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764241.084236:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764241.084237:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764241.084239:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764241.084240:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764241.084245:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764241.084247:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764241.084250:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764241.084307:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764241.084311:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764241.084314:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764241.468281:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764241.468289:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764241.468292:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764241.468298:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764241.468302:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764241.468306:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764241.468308:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764241.468311:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764241.468314:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764242.108248:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764242.108254:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764242.108256:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764242.108256:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764242.108262:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764242.108263:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764242.108270:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764242.108272:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764242.108274:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764242.108275:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764242.108279:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764242.108283:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764242.108285:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764242.108286:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764242.108292:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764242.108339:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764242.108346:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764242.108352:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764242.492271:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764242.492284:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764242.492287:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764242.492295:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764242.492301:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764242.492305:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764242.492309:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764242.492313:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764242.492317:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764243.132330:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764243.132339:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764243.132358:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764243.132362:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764243.132364:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764243.132375:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764243.132376:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764243.132382:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764243.132384:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764243.132391:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764243.132393:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764243.132397:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764243.132401:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764243.132403:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764243.132424:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764243.132430:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764243.132435:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764243.132439:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764243.132441:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764243.132443:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764243.132445:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764243.132562:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764243.132570:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764243.132574:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764243.516259:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764243.516270:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764243.516273:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764243.516282:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764243.516287:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764243.516292:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764243.516295:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764243.516298:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764243.516301:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764244.156268:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764244.156272:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764244.156274:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764244.156280:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764244.156281:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764244.156285:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764244.156286:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764244.156289:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764244.156291:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764244.156292:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764244.156299:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764244.156304:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764244.156308:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764244.156311:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764244.156315:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764244.156350:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764244.156356:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764244.156360:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764244.540302:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:2.0:1787764244.540303:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764244.540311:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:2.0:1787764244.540312:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764244.540314:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764244.540316:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:3.0:1787764244.540322:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:2.0:1787764244.540323:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764244.540330:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764244.540330:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764244.540335:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1787764244.540339:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21959
00000100:00000001:3.0:1787764244.540340:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764244.540345:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764244.540348:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764244.540353:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764244.540359:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764244.540362:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764244.540365:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764244.540367:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764244.540370:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764244.540373:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764244.540376:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764244.540378:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787764244.540381:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764244.540384:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787764244.540385:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787764244.540387:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787764244.540390:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764244.540391:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764244.540392:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764244.540394:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764244.540396:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787764244.540399:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764244.540401:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764244.540403:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764244.540407:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764244.540409:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764244.540410:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764244.540412:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764244.540421:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000020:00000040:2.0:1787764244.540424:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787764244.540428:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764244.540430:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764244.540433:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764244.540435:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764244.540440:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764244.540449:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff975202074c00.
00000100:00000001:2.0:1787764244.540456:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764244.540466:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832705408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764244.540479:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764244.540484:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764244.540490:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832705408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764244.540512:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764244.540515:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764244.540518:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787764244.540519:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764244.540520:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764244.540523:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764244.540525:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21959
00000400:00000200:2.0:1787764244.540529:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764244.540533:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764244.540535:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787764244.540546:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1787764244.540551:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764244.540552:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764244.540556:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1787764244.540557:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764244.540561:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764244.540564:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:3.0:1787764244.540566:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832705408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787764244.540567:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764244.540571:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:2.0:1787764244.540574:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764244.540578:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:2.0:1787764244.540582:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:3.0:1787764244.540582:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764244.540586:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764244.540587:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:3.0:1787764244.540591:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:2.0:1787764244.540593:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764244.540596:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1787764244.540596:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764244.540601:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764244.540602:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764244.540608:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764244.540612:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00100000:3.0:1787764244.540612:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832705408:192.168.203.110@tcp:250:kworker.0
00000100:00080000:2.0:1787764244.540619:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764244.540621:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1787764244.540628:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764244.540629:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764244.540631:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764244.540634:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764244.540638:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764244.540641:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1787764244.540642:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764244.540646:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764244.540650:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000010:3.0:1787764244.540651:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
00000008:00000001:2.0:1787764244.540653:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764244.540656:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764244.540657:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764244.540664:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000020:00000040:2.0:1787764244.540668:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000200:3.0:1787764244.540669:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:2.0:1787764244.540671:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764244.540674:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764244.540675:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:3.0:1787764244.540677:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
02000000:00000001:2.0:1787764244.540678:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764244.540681:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764244.540687:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff975202077400.
00000100:00000001:2.0:1787764244.540690:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764244.540690:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00100000:2.0:1787764244.540698:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832705536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1787764244.540698:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832705408, portal 25
00000100:00000001:3.0:1787764244.540705:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764244.540708:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000001:2.0:1787764244.540710:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764244.540713:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764244.540716:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832705536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787764244.540729:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832705408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764269 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764244.540731:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764244.540739:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764244.540741:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:2.0:1787764244.540746:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000040:3.0:1787764244.540747:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764244.540752:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000001:2.0:1787764244.540753:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000200:3.0:1787764244.540755:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832705408, offset 0
00000100:00080000:2.0:1787764244.540757:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764244.540762:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1787764244.540763:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00080000:2.0:1787764244.540770:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21959
00000400:00000200:3.0:1787764244.540774:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764244.540783:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764244.540788:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764244.540792:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764244.540795:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764244.540797:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764244.540800:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58746:58746:256:4294967295] 192.168.203.110@tcp LPNI seq info [58746:58746:8:4294967295]
00000400:00000200:3.0:1787764244.540804:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764244.540810:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764244.540815:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764244.540817:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000010:3.0:1787764244.540821:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000400:00000200:2.0:1787764244.540823:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764244.540827:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000200:3.0:1787764244.540831:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:2.0:1787764244.540832:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764244.540837:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1787764244.540838:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000100:00000040:2.0:1787764244.540843:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764244.540847:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:3.1:1787764244.540848:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:2.0:1787764244.540852:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:3.1:1787764244.540854:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000020:00000040:2.0:1787764244.540855:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787764244.540859:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764244.540862:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000100:3.0:1787764244.540862:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000020:00000001:2.0:1787764244.540866:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000010:3.0:1787764244.540867:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000020:00000001:2.0:1787764244.540870:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:2.0:1787764244.540875:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:3.0:1787764244.540875:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1787764244.540878:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764244.540884:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:3.0:1787764244.540887:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1787764244.540888:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764244.540892:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:3.0:1787764244.540893:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000100:00080000:2.0:1787764244.540897:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000010:3.0:1787764244.540899:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:2.0:1787764244.540902:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764244.540905:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764244.540905:0:257575:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:2.0:1787764244.540908:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:3.0:1787764244.540912:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832705408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764269 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764244.540913:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764244.540918:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764244.540921:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787764244.540926:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764244.540929:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764244.540931:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000010:2.0:1787764244.540934:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000100:00000040:3.0:1787764244.540936:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832705408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764269 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787764244.540938:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764244.540943:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764244.540945:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764244.540950:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1787764244.540952:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764244.540953:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1787764244.540956:0:257575:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:2.0:1787764244.540957:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764244.540962:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff975202077800.
00000100:00000001:3.0:1787764244.540963:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764244.540966:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764244.540970:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764244.540973:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316632819
00000100:00000010:3.0:1787764244.540978:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d403c0.
00000100:00100000:2.0:1787764244.540980:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832705664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764244.540985:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787764244.540990:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764244.540993:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764244.540997:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316632819
00000100:00000040:2.0:1787764244.540998:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832705664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1787764244.541004:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40180.
00000100:00000001:3.0:1787764244.541009:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787764244.541013:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764244.541015:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1787764244.541017:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764244.541018:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764244.541023:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1787764244.541024:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00080000:2.0:1787764244.541030:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:3.0:1787764244.541033:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764244/real 1787764244]  req@ffff9751f865ad00 x1874599832705408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787764244.541035:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787764244.541038:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764244.541041:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764244.541047:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000400:00000001:3.0:1787764244.541052:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00080000:2.0:1787764244.541053:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21959
00000400:00000001:3.0:1787764244.541055:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764244.541058:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:2.0:1787764244.541060:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764244.541062:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000200:2.0:1787764244.541066:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000010:3.0:1787764244.541066:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00080000:2.0:1787764244.541070:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787764244.541071:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:2.0:1787764244.541077:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764244.541080:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:3.0:1787764244.541080:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832705408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787764244.541089:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764244.541091:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764244.541096:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764244.541099:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787764244.541102:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764244.541107:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787764244.541111:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000200:3.0:1787764244.541111:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832705408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787764244.541116:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:2.0:1787764244.541123:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764244.541127:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764244.541132:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764244.541163:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764244.541166:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764244.541169:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764244.541170:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764244.541175:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832705408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.541182:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764244.541189:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832705408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764244.541198:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764244.541198:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000040:2.0:1787764244.541205:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:3.0:1787764244.541205:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764244.541209:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00080000:2.0:1787764244.541211:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764244.541212:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764244.541217:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764244.541219:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
02000000:00000001:2.0:1787764244.541221:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764244.541224:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:2.0:1787764244.541225:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:3.0:1787764244.541228:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000001:2.0:1787764244.541230:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764244.541235:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00080000:3.0:1787764244.541235:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000008:00100000:2.0:1787764244.541241:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1787764244.541242:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000008:00000001:2.0:1787764244.541245:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764244.541249:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764244.541250:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832705408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1787764244.541253:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
00000020:00000040:2.0:1787764244.541259:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:2.0:1787764244.541265:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00100000:3.0:1787764244.541266:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832705408:192.168.203.110@tcp:250:kworker.0
02000000:00000001:2.0:1787764244.541267:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764244.541272:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764244.541275:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1787764244.541275:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1787764244.541277:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764244.541279:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000010:2.0:1787764244.541282:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff975202074000.
00000100:00000040:3.0:1787764244.541283:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832705408/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764244.541287:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764244.541295:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
00000100:00100000:2.0:1787764244.541296:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832705792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1787764244.541299:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764244.541303:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
00000100:00000001:2.0:1787764244.541306:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:3.0:1787764244.541308:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000001:2.0:1787764244.541309:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787764244.541311:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764244.541316:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000100:00000040:2.0:1787764244.541317:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832705792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:3.0:1787764244.541321:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764244.541325:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff975202074c00.
02000000:00000001:3.0:1787764244.541330:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764244.541333:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764244.541336:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:2.0:1787764244.541339:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764244.541341:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764244.541343:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764244.541344:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21969)
00000100:00000001:3.0:1787764244.541346:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764244.541354:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764244.541355:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764244.541357:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764244.541359:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764244.541361:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764244.541365:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832705536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.541376:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764244.541377:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764244.541379:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764244.541380:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764244.541382:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764244.541384:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832705536:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764244.541386:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764244.541387:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764244.541389:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764244.541390:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764244.541392:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764244.541394:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764244.541396:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764244.541398:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764244.541401:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764244.541403:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832705536, portal 10
00000100:00000001:3.0:1787764244.541405:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764244.541406:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:3.0:1787764244.541411:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832705536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764269 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.541416:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764244.541417:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764244.541419:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764244.541421:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832705536, offset 0
00000400:00000200:3.0:1787764244.541423:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764244.541426:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764244.541429:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764244.541431:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764244.541433:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764244.541435:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764244.541437:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764244.541439:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58747:58747:256:4294967295] 192.168.203.110@tcp LPNI seq info [58747:58747:8:4294967295]
00000400:00000200:3.0:1787764244.541450:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764244.541455:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764244.541459:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764244.541461:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764244.541464:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764244.541467:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787764244.541470:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764244.541472:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1787764244.541474:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764244.541476:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764244.541478:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764244.541484:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764244.541487:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764244.541488:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764244.541490:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764244.541493:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832705536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764269 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.541499:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764244.541501:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832705536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764269 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.541506:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764244.541507:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764244.541509:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764244.541512:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764244.541515:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764244.541516:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316631584
00000100:00000001:3.0:1787764244.541518:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316632819
00000100:00000001:3.0:1787764244.541520:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316631584
00000100:00000001:3.0:1787764244.541522:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316632819
00000100:00000001:3.0:1787764244.541523:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764244.541525:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764244.541527:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764244.541529:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764244/real 1787764244]  req@ffff9751f86583c0 x1874599832705536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764244.541535:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764244.541550:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764244.541553:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764244.541556:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764244.541557:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764244.541559:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764244.541562:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832705536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764244.541568:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832705536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.541573:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764244.541574:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764244.541575:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764244.541576:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764244.541579:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832705536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.541583:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764244.541586:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832705536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.541591:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764244.541593:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764244.541594:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764244.541595:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764244.541596:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764244.541606:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764244.541607:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764244.541610:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764244.541612:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764244.541615:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832705536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764244.541620:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832705536:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764244.541623:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764244.541625:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832705664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.541630:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764244.541631:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764244.541633:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764244.541635:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764244.541637:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764244.541640:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832705664:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764244.541644:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764244.541646:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764244.541648:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764244.541650:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764244.541654:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764244.541656:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764244.541660:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764244.541663:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764244.541667:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764244.541670:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832705664, portal 10
00000100:00000001:3.0:1787764244.541672:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764244.541674:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:3.0:1787764244.541682:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832705664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764269 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.541689:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764244.541693:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764244.541695:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764244.541698:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832705664, offset 0
00000400:00000200:3.0:1787764244.541702:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764244.541705:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764244.541709:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764244.541711:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764244.541713:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764244.541715:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764244.541717:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764244.541719:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58748:58748:256:4294967295] 192.168.203.110@tcp LPNI seq info [58748:58748:8:4294967295]
00000400:00000200:3.0:1787764244.541722:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764244.541727:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764244.541730:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764244.541733:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764244.541736:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764244.541738:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787764244.541741:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764244.541744:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1787764244.541746:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764244.541748:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764244.541750:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764244.541753:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764244.541755:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764244.541756:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764244.541758:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764244.541761:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832705664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764269 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.541767:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764244.541769:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832705664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764269 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.541774:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764244.541775:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764244.541778:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764244.541780:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764244.541782:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764244.541784:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316631572
00000100:00000001:3.0:1787764244.541785:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316632819
00000100:00000001:3.0:1787764244.541787:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316631572
00000100:00000001:3.0:1787764244.541789:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316632819
00000100:00000001:3.0:1787764244.541790:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764244.541792:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764244.541793:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764244.541796:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764244/real 1787764244]  req@ffff9751f8659a40 x1874599832705664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764244.541801:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764244.541802:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764244.541803:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764244.541805:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764244.541806:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764244.541808:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764244.541810:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832705664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764244.541816:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832705664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.541820:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764244.541821:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764244.541822:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764244.541823:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764244.541825:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832705664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.541830:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764244.541832:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832705664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.541838:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764244.541840:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764244.541841:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764244.541842:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764244.541843:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764244.541844:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764244.541846:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764244.541848:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764244.541849:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764244.541852:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832705664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764244.541857:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832705664:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764244.541859:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764244.541861:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832705792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.541866:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764244.541867:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764244.541868:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764244.541869:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764244.541870:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764244.541872:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832705792:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764244.541874:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764244.541876:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764244.541877:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764244.541878:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764244.541881:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764244.541882:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764244.541885:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764244.541887:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764244.541889:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764244.541891:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832705792, portal 4
00000100:00000001:3.0:1787764244.541893:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764244.541894:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:3.0:1787764244.541898:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832705792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764269 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.541903:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764244.541904:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764244.541906:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764244.541908:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832705792, offset 0
00000400:00000200:3.0:1787764244.541910:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764244.541912:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764244.541914:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764244.541916:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764244.541918:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764244.541920:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764244.541921:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764244.541924:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58749:58749:256:4294967295] 192.168.203.110@tcp LPNI seq info [58749:58749:8:4294967295]
00000400:00000200:3.0:1787764244.541927:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764244.541931:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764244.541934:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764244.541937:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764244.541939:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764244.541941:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000200:3.1:1787764244.541944:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764244.541946:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 42 secs later)
00000800:00000100:3.0:1787764244.541949:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764244.541950:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764244.541952:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764244.541955:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764244.541957:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764244.541959:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764244.541960:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764244.541963:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832705792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764269 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.541968:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764244.541970:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832705792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764269 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.541974:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764244.541975:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764244.541977:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764244.541979:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764244.541982:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764244.541983:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316631589
00000100:00000001:3.0:1787764244.541985:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316632819
00000100:00000001:3.0:1787764244.541987:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316631589
00000100:00000001:3.0:1787764244.541988:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316632819
00000100:00000001:3.0:1787764244.541990:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764244.541991:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764244.541992:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764244.541994:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764244/real 1787764244]  req@ffff9751f8658000 x1874599832705792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764244.541999:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764244.542000:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764244.542001:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764244.542002:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764244.542004:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764244.542005:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764244.542007:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832705792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764244.542013:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832705792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.542018:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764244.542019:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764244.542020:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764244.542021:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764244.542024:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832705792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.542028:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764244.542030:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832705792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.542035:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764244.542036:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764244.542037:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764244.542038:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764244.542039:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764244.542040:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764244.542042:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764244.542044:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764244.542045:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764244.542048:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832705792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764244.542052:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832705792:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764244.542055:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764244.542056:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764244.542058:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832705536/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.542063:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764244.542064:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764244.542066:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764244.542068:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764244.542069:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764244.542070:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764244.542072:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764244.542073:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff975202077400.
02000000:00000001:3.0:1787764244.542075:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764244.542076:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764244.542077:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:3.0:1787764244.542078:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764244.542080:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764244.542081:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764244.542083:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832705664/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.542087:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764244.542088:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764244.542089:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764244.542091:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764244.542091:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764244.542092:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764244.542093:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764244.542094:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff975202077800.
02000000:00000001:3.0:1787764244.542096:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764244.542096:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764244.542097:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:3.0:1787764244.542099:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764244.542099:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764244.542100:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764244.542102:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832705792/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764269 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764244.542106:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764244.542107:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764244.542108:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764244.542109:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764244.542110:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764244.542111:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764244.542113:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764244.542114:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff975202074000.
02000000:00000001:3.0:1787764244.542115:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764244.542116:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764244.542117:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:3.0:1787764244.542118:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764244.542119:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764244.542120:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764244.542123:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764244.542124:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764244.542125:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764244.542126:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764245.180241:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764245.180246:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764245.180248:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764245.180253:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764245.180254:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764245.180260:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764245.180263:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764245.180264:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764245.180268:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764245.180269:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764245.180271:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764245.180274:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764245.180277:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764245.180278:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764245.180282:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764245.180330:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764245.180337:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764245.180341:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764245.564253:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764245.564259:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764245.564261:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764245.564266:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764245.564271:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764245.564274:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764245.564277:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764245.564279:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764245.564281:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764245.820245:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316632819, at 4316632832
00000100:00000010:3.0:1787764245.820261:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40180.
00000100:00000040:2.0:1787764245.820290:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316632819, at 4316632832
00000100:00000010:2.0:1787764245.820301:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d403c0.
00000100:00000001:2.0:1787764246.204276:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764246.204286:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764246.204288:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764246.204291:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764246.204294:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764246.204297:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764246.204301:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764246.204302:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764246.204307:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764246.204308:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764246.204315:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764246.204316:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764246.204320:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764246.204321:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764246.204328:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764246.204328:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764246.204333:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764246.204341:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764246.204343:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764246.204348:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764246.204349:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764246.204416:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764246.204421:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764246.204425:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764246.588299:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764246.588307:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764246.588311:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764246.588318:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764246.588322:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764246.588326:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764246.588328:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764246.588331:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764246.588334:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764247.228222:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764247.228233:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764247.228236:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764247.228236:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764247.228242:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764247.228246:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764247.228249:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764247.228253:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764247.228253:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764247.228259:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764247.228260:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764247.228262:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764247.228265:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764247.228271:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764247.228275:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764247.228346:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764247.228352:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764247.228356:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764247.612244:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764247.612251:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764247.612253:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764247.612257:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764247.612260:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764247.612263:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764247.612266:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764247.612268:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764247.612270:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764248.252232:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764248.252240:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764248.252242:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764248.252246:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764248.252247:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764248.252255:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764248.252255:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764248.252263:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764248.252263:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764248.252268:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764248.252270:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764248.252274:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764248.252275:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764248.252281:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764248.252281:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764248.252286:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764248.252287:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764248.252291:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764248.252296:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764248.252302:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764248.252304:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764248.252355:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764248.252363:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764248.252368:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764248.636236:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764248.636244:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764248.636246:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764248.636252:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764248.636256:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764248.636260:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764248.636263:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764248.636271:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764248.636276:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764249.276259:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764249.276265:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764249.276268:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764249.276271:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764249.276273:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764249.276284:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764249.276285:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764249.276287:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764249.276292:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764249.276296:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764249.276297:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764249.276301:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764249.276302:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764249.276308:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764249.276312:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764249.276372:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764249.276378:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764249.276382:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764249.660259:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:2.0:1787764249.660268:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764249.660269:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764249.660273:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764249.660280:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764249.660283:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764249.660288:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764249.660293:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:2.0:1787764249.660296:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764249.660296:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764249.660303:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1787764249.660303:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764249.660313:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764249.660321:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764249.660331:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21964
00000400:00000200:2.0:1787764249.660340:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764249.660346:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764249.660350:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764249.660352:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764249.660355:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764249.660357:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764249.660359:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764249.660363:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764249.660364:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787764249.660367:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764249.660371:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787764249.660372:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787764249.660374:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787764249.660377:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764249.660379:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764249.660381:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764249.660382:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764249.660385:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787764249.660387:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764249.660389:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764249.660391:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764249.660395:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764249.660397:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764249.660399:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764249.660401:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764249.660410:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658780.
00000020:00000040:2.0:1787764249.660413:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787764249.660417:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764249.660418:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764249.660420:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764249.660424:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764249.660428:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764249.660435:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff975202077400.
00000100:00000001:2.0:1787764249.660441:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764249.660451:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658780 x1874599832705920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764249.660460:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764249.660464:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764249.660468:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000744875b5] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658780 x1874599832705920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764249.660489:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764249.660491:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764249.660494:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787764249.660496:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764249.660497:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764249.660515:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764249.660519:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21964
00000400:00000200:2.0:1787764249.660524:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764249.660529:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787764249.660529:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:2.0:1787764249.660533:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787764249.660536:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764249.660539:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764249.660541:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764249.660548:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764249.660552:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764249.660558:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:3.0:1787764249.660559:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832705920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787764249.660565:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764249.660571:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:2.0:1787764249.660576:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764249.660582:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000001:3.0:1787764249.660583:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:2.0:1787764249.660587:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:3.0:1787764249.660589:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764249.660595:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:3.0:1787764249.660596:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:2.0:1787764249.660605:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:3.0:1787764249.660605:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764249.660621:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1787764249.660622:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764249.660641:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00100000:3.0:1787764249.660641:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832705920:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764249.660653:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764249.660655:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000040:2.0:1787764249.660661:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:3.0:1787764249.660663:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764249.660666:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764249.660667:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:3.0:1787764249.660673:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1787764249.660675:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764249.660681:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:3.0:1787764249.660685:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:2.0:1787764249.660689:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764249.660689:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764249.660695:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764249.660697:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1787764249.660700:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000008:00100000:2.0:1787764249.660703:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000010:3.0:1787764249.660706:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000008:00000001:2.0:1787764249.660707:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764249.660713:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764249.660718:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000010:2.0:1787764249.660719:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
00000020:00000040:2.0:1787764249.660728:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000100:00000200:3.0:1787764249.660733:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832705920, portal 25
02000000:00000001:2.0:1787764249.660735:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764249.660737:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764249.660737:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764249.660743:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000001:2.0:1787764249.660744:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764249.660748:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764249.660751:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764249.660756:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832705920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764274 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1787764249.660759:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff975202074c00.
00000100:00000001:2.0:1787764249.660770:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764249.660771:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764249.660777:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764249.660780:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764249.660785:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832705920, offset 0
00000100:00100000:2.0:1787764249.660786:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832706048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787764249.660798:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787764249.660813:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1787764249.660814:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764249.660821:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764249.660825:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000040:2.0:1787764249.660830:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832706048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764249.660834:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764249.660847:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764249.660855:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764249.660861:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:2.0:1787764249.660864:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764249.660867:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58750:58750:256:4294967295] 192.168.203.110@tcp LPNI seq info [58750:58750:8:4294967295]
00000100:00080000:2.0:1787764249.660874:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1787764249.660875:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:2.0:1787764249.660885:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1787764249.660890:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1787764249.660892:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00080000:2.0:1787764249.660894:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764249.660900:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000800:00000200:3.0:1787764249.660900:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00080000:2.0:1787764249.660910:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21964
00000800:00000010:3.0:1787764249.660911:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000400:00000200:2.0:1787764249.660920:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:3.0:1787764249.660926:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000400:00000200:2.0:1787764249.660929:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764249.660935:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000200:3.0:1787764249.660936:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000100:00000040:2.0:1787764249.660945:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:3.1:1787764249.660954:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787764249.660955:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.1:1787764249.660963:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000100:00000040:2.0:1787764249.660966:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764249.660970:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000100:3.0:1787764249.660972:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000020:00000001:2.0:1787764249.660979:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000010:3.0:1787764249.660979:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000020:00000040:2.0:1787764249.660986:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787764249.660989:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:3.0:1787764249.660989:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:2.0:1787764249.660994:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:2.0:1787764249.661001:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:3.0:1787764249.661002:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:2.0:1787764249.661010:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:3.0:1787764249.661014:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000100:00000040:2.0:1787764249.661015:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000010:3.0:1787764249.661019:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:2.0:1787764249.661021:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764249.661026:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:2.0:1787764249.661027:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764249.661035:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000200:3.0:1787764249.661039:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832705920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764274 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787764249.661041:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:2.0:1787764249.661052:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764249.661064:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764249.661074:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764249.661078:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1787764249.661081:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1787764249.661085:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832705920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764274 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764249.661090:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764249.661101:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764249.661101:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764249.661107:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000008:00100000:2.0:1787764249.661109:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787764249.661112:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1787764249.661116:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764249.661122:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764249.661126:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764249.661128:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316637939
00000100:00000010:2.0:1787764249.661156:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000020:00000040:2.0:1787764249.661160:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764249.661163:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764249.661165:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764249.661168:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000010:3.0:1787764249.661169:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40180.
02000000:00000001:2.0:1787764249.661172:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764249.661180:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764249.661181:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
02000000:00000010:2.0:1787764249.661193:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff975202077800.
00000100:00000001:3.0:1787764249.661193:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316637940
00000100:00000010:3.0:1787764249.661202:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40360.
00000100:00000001:2.0:1787764249.661204:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764249.661208:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787764249.661213:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:2.0:1787764249.661220:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832706176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764249.661220:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764249.661228:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764249.661237:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764249/real 1787764249]  req@ffff9751f8658780 x1874599832705920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764249.661240:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764249.661246:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764249.661262:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832706176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764249.661263:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764249.661268:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764249.661273:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764249.661278:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764249.661281:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:2.0:1787764249.661284:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764249.661285:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:2.0:1787764249.661292:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:3.0:1787764249.661292:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832705920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787764249.661313:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000200:3.0:1787764249.661319:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832705920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787764249.661321:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764249.661329:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00080000:2.0:1787764249.661331:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787764249.661332:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764249.661335:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1787764249.661336:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00080000:2.0:1787764249.661338:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764249.661339:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764249.661344:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00100000:3.0:1787764249.661349:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832705920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787764249.661351:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21964
00000400:00000200:2.0:1787764249.661369:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764249.661382:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787764249.661388:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1787764249.661389:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764249.661402:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000040:3.0:1787764249.661404:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832705920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764249.661409:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764249.661420:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764249.661430:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764249.661435:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000001:2.0:1787764249.661436:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764249.661439:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000100:00000001:3.0:1787764249.661441:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1787764249.661443:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787764249.661446:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000040:2.0:1787764249.661449:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000001:3.0:1787764249.661450:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1787764249.661459:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:3.0:1787764249.661461:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764249.661467:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000001:2.0:1787764249.661468:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00080000:3.0:1787764249.661486:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000040:2.0:1787764249.661488:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764249.661500:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00080000:3.0:1787764249.661515:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000040:2.0:1787764249.661518:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764249.661521:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:2.0:1787764249.661522:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764249.661530:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000040:3.0:1787764249.661536:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832705920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787764249.661541:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764249.661551:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764249.661559:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764249.661566:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:3.0:1787764249.661570:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832705920:192.168.203.110@tcp:250:kworker.0
02000000:00000001:2.0:1787764249.661572:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764249.661580:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764249.661585:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1787764249.661588:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000008:00000001:2.0:1787764249.661589:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764249.661592:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787764249.661593:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764249.661601:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000100:00000040:3.0:1787764249.661602:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832705920/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787764249.661611:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:2.0:1787764249.661621:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1787764249.661622:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:2.0:1787764249.661623:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764249.661626:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000100:00000001:2.0:1787764249.661627:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764249.661631:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000010:3.0:1787764249.661631:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:2.0:1787764249.661639:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764249.661641:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
02000000:00000010:2.0:1787764249.661646:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff975202074000.
00000020:00000001:3.0:1787764249.661646:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764249.661650:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000100:00000001:2.0:1787764249.661652:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:3.0:1787764249.661654:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764249.661665:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff975202077400.
00000100:00100000:2.0:1787764249.661668:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832706304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1787764249.661670:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764249.661675:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764249.661681:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:2.0:1787764249.661691:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764249.661692:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:2.0:1787764249.661698:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764249.661698:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764249.661713:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764249.661719:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832706304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.661724:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764249.661730:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764249.661731:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764249.661744:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764249.661748:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764249.661749:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764249.661751:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21974)
00000100:00000040:3.0:1787764249.661756:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832706048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.661763:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764249.661765:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764249.661766:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764249.661768:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764249.661771:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764249.661773:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832706048:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764249.661776:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764249.661779:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764249.661780:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764249.661781:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764249.661784:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764249.661785:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764249.661788:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764249.661790:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764249.661793:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764249.661795:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832706048, portal 10
00000100:00000001:3.0:1787764249.661796:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764249.661797:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:3.0:1787764249.661802:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832706048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764274 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.661806:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764249.661808:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764249.661809:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764249.661811:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832706048, offset 0
00000400:00000200:3.0:1787764249.661813:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764249.661816:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764249.661819:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764249.661821:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764249.661823:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764249.661825:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764249.661826:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764249.661829:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58751:58751:256:4294967295] 192.168.203.110@tcp LPNI seq info [58751:58751:8:4294967295]
00000400:00000200:3.0:1787764249.661832:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764249.661836:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764249.661839:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764249.661841:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764249.661844:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764249.661845:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787764249.661850:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764249.661851:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1787764249.661854:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764249.661856:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764249.661858:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764249.661860:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764249.661862:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764249.661864:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764249.661865:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764249.661867:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832706048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764274 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.661872:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764249.661874:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832706048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764274 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.661879:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764249.661880:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764249.661881:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764249.661884:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764249.661886:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764249.661887:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316636704
00000100:00000001:3.0:1787764249.661889:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316637939
00000100:00000001:3.0:1787764249.661891:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316636704
00000100:00000001:3.0:1787764249.661892:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316637940
00000100:00000001:3.0:1787764249.661894:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764249.661895:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764249.661896:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764249.661898:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764249/real 1787764249]  req@ffff9751f8658000 x1874599832706048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764249.661903:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764249.661904:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764249.661905:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764249.661906:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764249.661907:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764249.661909:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764249.661911:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832706048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764249.661916:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832706048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.661920:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764249.661921:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764249.661922:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764249.661923:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764249.661925:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832706048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.661929:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764249.661932:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832706048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.661936:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764249.661937:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764249.661938:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764249.661939:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764249.661940:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764249.661950:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764249.661951:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764249.661954:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764249.661955:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764249.661958:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832706048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764249.661963:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832706048:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764249.661965:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764249.661967:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832706176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.661976:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764249.661977:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764249.661979:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764249.661980:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764249.661981:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764249.661983:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832706176:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764249.661985:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764249.661986:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764249.661987:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764249.661988:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764249.661991:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764249.661992:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764249.661994:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764249.661996:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764249.661997:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764249.661999:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832706176, portal 10
00000100:00000001:3.0:1787764249.662000:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764249.662001:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:3.0:1787764249.662005:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832706176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764274 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.662010:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764249.662011:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764249.662012:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764249.662014:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832706176, offset 0
00000400:00000200:3.0:1787764249.662016:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764249.662018:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764249.662020:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764249.662022:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764249.662023:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764249.662025:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764249.662026:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764249.662028:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58752:58752:256:4294967295] 192.168.203.110@tcp LPNI seq info [58752:58752:8:4294967295]
00000400:00000200:3.0:1787764249.662031:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764249.662035:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764249.662038:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764249.662040:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764249.662042:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764249.662043:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787764249.662046:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764249.662048:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1787764249.662050:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764249.662052:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764249.662053:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764249.662056:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764249.662058:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764249.662059:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764249.662060:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764249.662062:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832706176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764274 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.662067:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764249.662069:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832706176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764274 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.662073:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764249.662074:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764249.662076:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764249.662077:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764249.662079:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764249.662080:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316636692
00000100:00000001:3.0:1787764249.662081:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316637939
00000100:00000001:3.0:1787764249.662083:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316636692
00000100:00000001:3.0:1787764249.662084:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316637940
00000100:00000001:3.0:1787764249.662086:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764249.662087:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764249.662089:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764249.662091:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764249/real 1787764249]  req@ffff9751f8659a40 x1874599832706176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764249.662096:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764249.662097:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764249.662098:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764249.662099:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764249.662100:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764249.662101:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764249.662103:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832706176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764249.662108:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832706176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.662112:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764249.662113:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764249.662114:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764249.662115:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764249.662117:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832706176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.662121:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764249.662123:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832706176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.662127:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764249.662128:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764249.662129:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764249.662130:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764249.662131:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764249.662149:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764249.662151:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764249.662154:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764249.662155:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764249.662158:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832706176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764249.662162:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832706176:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764249.662165:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764249.662167:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832706304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.662172:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764249.662173:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764249.662174:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764249.662175:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764249.662177:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764249.662178:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832706304:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764249.662180:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764249.662182:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764249.662183:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764249.662184:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764249.662187:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764249.662188:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764249.662190:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764249.662192:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764249.662194:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764249.662196:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832706304, portal 4
00000100:00000001:3.0:1787764249.662197:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764249.662198:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:3.0:1787764249.662202:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832706304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764274 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.662206:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764249.662208:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764249.662209:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764249.662211:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832706304, offset 0
00000400:00000200:3.0:1787764249.662213:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764249.662215:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764249.662218:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764249.662219:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764249.662221:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764249.662223:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764249.662224:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764249.662226:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58753:58753:256:4294967295] 192.168.203.110@tcp LPNI seq info [58753:58753:8:4294967295]
00000400:00000200:3.0:1787764249.662229:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764249.662233:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764249.662235:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764249.662237:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764249.662239:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764249.662241:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000200:3.1:1787764249.662244:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764249.662245:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 37 secs later)
00000800:00000100:3.0:1787764249.662248:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764249.662249:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764249.662251:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764249.662253:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764249.662255:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764249.662256:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764249.662258:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764249.662260:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832706304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764274 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.662265:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764249.662267:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832706304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764274 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.662271:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764249.662272:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764249.662274:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764249.662276:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764249.662278:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764249.662280:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316636710
00000100:00000001:3.0:1787764249.662281:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316637939
00000100:00000001:3.0:1787764249.662283:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316636710
00000100:00000001:3.0:1787764249.662285:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316637940
00000100:00000001:3.0:1787764249.662286:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764249.662287:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764249.662288:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764249.662290:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764249/real 1787764249]  req@ffff9751f86583c0 x1874599832706304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764249.662294:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764249.662295:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764249.662296:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764249.662298:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764249.662299:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764249.662300:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764249.662302:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832706304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764249.662307:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832706304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.662311:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764249.662312:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764249.662313:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764249.662314:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764249.662316:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832706304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.662320:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764249.662322:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832706304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.662327:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764249.662328:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764249.662329:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764249.662330:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764249.662331:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764249.662332:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764249.662334:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764249.662336:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764249.662337:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764249.662339:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832706304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764249.662344:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832706304:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764249.662346:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764249.662347:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764249.662349:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832706048/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.662352:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764249.662353:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764249.662354:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764249.662356:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764249.662357:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764249.662357:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764249.662359:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764249.662360:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff975202074c00.
02000000:00000001:3.0:1787764249.662361:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764249.662362:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764249.662363:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:3.0:1787764249.662365:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764249.662366:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764249.662367:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764249.662369:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832706176/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.662374:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764249.662375:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764249.662375:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764249.662377:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764249.662377:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764249.662378:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764249.662379:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764249.662380:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff975202077800.
02000000:00000001:3.0:1787764249.662381:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764249.662382:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764249.662383:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:3.0:1787764249.662384:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764249.662385:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764249.662386:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764249.662387:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832706304/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764274 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764249.662391:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764249.662392:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764249.662392:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764249.662394:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764249.662394:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764249.662395:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764249.662396:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764249.662397:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff975202074000.
02000000:00000001:3.0:1787764249.662398:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764249.662399:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764249.662399:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:3.0:1787764249.662401:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764249.662401:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764249.662402:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764249.662404:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764249.662405:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764249.662406:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764249.662408:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764250.300289:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764250.300291:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764250.300294:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764250.300304:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764250.300305:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764250.300308:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764250.300311:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764250.300313:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764250.300316:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764250.300320:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764250.300326:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764250.300331:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764250.300335:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764250.300338:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764250.300341:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764250.300361:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764250.300365:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764250.300368:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764250.684251:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764250.684260:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764250.684262:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764250.684269:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764250.684273:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764250.684277:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764250.684296:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764250.684299:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764250.684303:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764250.940263:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316637940, at 4316637952
00000100:00000010:3.0:1787764250.940278:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40360.
00000100:00000040:2.0:1787764250.940311:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316637939, at 4316637952
00000100:00000010:2.0:1787764250.940325:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40180.
00000100:00000001:2.0:1787764251.324214:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764251.324219:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764251.324222:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764251.324225:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764251.324227:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764251.324233:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764251.324234:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764251.324238:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764251.324239:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764251.324243:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764251.324246:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764251.324247:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764251.324248:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764251.324251:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764251.324252:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764251.324326:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764251.324332:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764251.324336:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764251.708243:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764251.708252:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764251.708254:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764251.708259:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764251.708265:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764251.708268:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764251.708270:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764251.708272:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764251.708275:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764252.348258:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764252.348260:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764252.348264:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764252.348266:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764252.348270:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764252.348271:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764252.348274:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764252.348275:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764252.348281:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764252.348284:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764252.348285:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764252.348289:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764252.348289:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764252.348291:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764252.348292:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764252.348296:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764252.348297:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764252.348301:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764252.348303:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764252.348305:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764252.348306:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764252.348347:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764252.348352:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764252.348356:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764252.732232:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764252.732239:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764252.732241:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764252.732245:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764252.732250:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764252.732254:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764252.732256:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764252.732259:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764252.732262:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764253.372215:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764253.372220:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764253.372222:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764253.372225:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764253.372226:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764253.372233:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764253.372234:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764253.372236:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764253.372238:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764253.372240:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764253.372242:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764253.372243:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764253.372248:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764253.372251:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764253.372253:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764253.372285:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764253.372288:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764253.372291:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764253.756275:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764253.756300:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764253.756303:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764253.756309:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764253.756313:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764253.756318:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764253.756321:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764253.756324:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764253.756327:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764254.396275:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764254.396275:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764254.396281:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764254.396283:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764254.396287:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764254.396289:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764254.396291:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764254.396291:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764254.396299:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764254.396299:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764254.396301:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764254.396304:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764254.396307:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764254.396309:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764254.396311:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764254.396369:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764254.396376:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764254.396382:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787764254.780247:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764254.780247:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764254.780254:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764254.780258:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764254.780260:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00100000:3.0:1787764254.780264:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764254.780265:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1787764254.780270:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:2.0:1787764254.780272:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764254.780274:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764254.780281:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764254.780281:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764254.780285:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764254.780288:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764254.780292:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21969
00000400:00000200:2.0:1787764254.780303:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764254.780309:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764254.780312:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764254.780316:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764254.780318:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764254.780321:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764254.780323:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764254.780326:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764254.780328:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787764254.780330:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764254.780333:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787764254.780334:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787764254.780336:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787764254.780338:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764254.780339:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764254.780341:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764254.780342:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764254.780344:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787764254.780346:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764254.780348:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764254.780350:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764254.780361:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764254.780364:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764254.780365:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764254.780368:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764254.780376:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000020:00000040:2.0:1787764254.780378:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787764254.780384:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764254.780385:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764254.780388:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764254.780391:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764254.780394:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764254.780401:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff975202077800.
00000100:00000001:2.0:1787764254.780406:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764254.780414:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832706432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764254.780424:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764254.780429:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764254.780434:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832706432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764254.780458:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764254.780461:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764254.780479:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787764254.780481:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764254.780483:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764254.780486:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764254.780488:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21969
00000100:00000001:3.0:1787764254.780489:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:2.0:1787764254.780493:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764254.780494:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000400:00000200:2.0:1787764254.780497:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787764254.780498:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00080000:2.0:1787764254.780502:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764254.780507:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000040:3.0:1787764254.780507:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832706432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764254.780509:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764254.780515:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764254.780521:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764254.780522:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764254.780525:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764254.780527:0:304484:0:(genops.c:792:class_conn2export()) Process entered
02000000:00000001:3.0:1787764254.780529:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000040:2.0:1787764254.780531:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
02000000:00000001:3.0:1787764254.780532:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:2.0:1787764254.780536:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:3.0:1787764254.780537:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1787764254.780542:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00100000:3.0:1787764254.780543:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832706432:192.168.203.110@tcp:250:kworker.0
00000020:00000001:2.0:1787764254.780549:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:3.0:1787764254.780550:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764254.780554:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000001:2.0:1787764254.780555:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:3.0:1787764254.780557:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764254.780560:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:2.0:1787764254.780564:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000010:3.0:1787764254.780566:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
00000100:00000001:2.0:1787764254.780568:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1787764254.780570:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764254.780574:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:3.0:1787764254.780578:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764254.780580:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000010:3.0:1787764254.780582:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000020:00000040:2.0:1787764254.780585:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:2.0:1787764254.780589:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000010:3.0:1787764254.780590:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000001:2.0:1787764254.780591:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764254.780594:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000200:3.0:1787764254.780597:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832706432, portal 25
02000000:00000001:2.0:1787764254.780600:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764254.780600:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764254.780603:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
02000000:00000001:2.0:1787764254.780604:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764254.780607:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:3.0:1787764254.780611:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832706432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:2.0:1787764254.780612:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787764254.780617:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764254.780621:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764254.780627:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000010:2.0:1787764254.780628:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000100:00000040:3.0:1787764254.780632:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000020:00000040:2.0:1787764254.780633:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000010:3.0:1787764254.780636:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
02000000:00000001:2.0:1787764254.780637:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764254.780640:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764254.780641:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832706432, offset 0
00000100:00000001:2.0:1787764254.780643:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764254.780645:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764254.780648:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764254.780650:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
02000000:00000010:2.0:1787764254.780655:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff975202074c00.
00000400:00000200:3.0:1787764254.780659:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764254.780661:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764254.780668:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00100000:2.0:1787764254.780671:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832706560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764254.780679:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764254.780681:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764254.780684:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:2.0:1787764254.780685:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832706560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764254.780693:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:2.0:1787764254.780695:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764254.780700:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1787764254.780700:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764254.780706:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00080000:2.0:1787764254.780707:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1787764254.780709:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764254.780712:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1787764254.780712:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58754:58754:256:4294967295] 192.168.203.110@tcp LPNI seq info [58754:58754:8:4294967295]
00000100:00000001:2.0:1787764254.780718:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1787764254.780721:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:2.0:1787764254.780723:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21969
00000400:00000200:3.0:1787764254.780731:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000400:00000200:2.0:1787764254.780738:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:3.0:1787764254.780739:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764254.780743:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000800:00000010:3.0:1787764254.780745:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000100:00080000:2.0:1787764254.780746:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764254.780751:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764254.780752:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1787764254.780756:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:2.0:1787764254.780757:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764254.780762:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:3.0:1787764254.780763:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000020:00000001:2.0:1787764254.780764:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764254.780766:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787764254.780768:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764254.780772:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000200:3.1:1787764254.780772:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:2.0:1787764254.780776:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:2.0:1787764254.780779:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:3.1:1787764254.780779:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000100:00000040:2.0:1787764254.780785:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764254.780789:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000100:3.0:1787764254.780790:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000040:2.0:1787764254.780793:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000010:3.0:1787764254.780795:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000100:00000001:2.0:1787764254.780796:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764254.780800:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:2.0:1787764254.780802:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764254.780804:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764254.780805:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1787764254.780806:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:2.0:1787764254.780807:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764254.780811:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764254.780813:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1787764254.780814:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764254.780819:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787764254.780821:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764254.780821:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764254.780825:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:2.0:1787764254.780826:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764254.780833:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000100:00000001:3.0:1787764254.780833:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000020:00000040:2.0:1787764254.780835:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764254.780839:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000200:3.0:1787764254.780840:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832706432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764254.780841:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764254.780846:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764254.780847:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764254.780850:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764254.780855:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff975202077400.
00000100:00000001:3.0:1787764254.780857:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787764254.780859:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764254.780863:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832706432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764279 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764254.780867:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832706688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764254.780881:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764254.780883:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764254.780884:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764254.780887:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764254.780891:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832706688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.780892:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764254.780901:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764254.780904:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764254.780905:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316643059
00000100:00000040:2.0:1787764254.780907:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1787764254.780911:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000010:3.0:1787764254.780912:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40360.
00000100:00080000:2.0:1787764254.780913:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764254.780917:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787764254.780919:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764254.780921:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764254.780921:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787764254.780926:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764254.780928:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21969
00000100:00000001:3.0:1787764254.780931:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316643059
00000400:00000200:2.0:1787764254.780933:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000010:3.0:1787764254.780936:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40c60.
00000400:00000200:2.0:1787764254.780938:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787764254.780940:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00080000:2.0:1787764254.780941:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787764254.780944:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1787764254.780946:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764254.780947:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764254.780949:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764254.780950:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000040:2.0:1787764254.780953:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764254.780957:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764254.780959:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764254.780960:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787764254.780961:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764254.780963:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787764254.780965:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00100000:3.0:1787764254.780965:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764254/real 1787764254]  req@ffff9751f865ad00 x1874599832706432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787764254.780976:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000001:3.0:1787764254.780979:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:2.0:1787764254.780980:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764254.780984:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000001:3.0:1787764254.780984:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:2.0:1787764254.780988:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000010:3.0:1787764254.780989:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000100:00000001:2.0:1787764254.780990:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764254.780996:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000400:00000200:3.0:1787764254.780997:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000100:00080000:2.0:1787764254.780999:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000010:3.0:1787764254.781002:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:2.0:1787764254.781003:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764254.781007:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764254.781009:0:257575:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:2.0:1787764254.781010:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764254.781014:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764254.781015:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764254.781018:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000200:3.0:1787764254.781019:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832706432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:2.0:1787764254.781020:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764254.781025:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764254.781030:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
00000020:00000040:2.0:1787764254.781035:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000200:3.0:1787764254.781038:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832706432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764254.781039:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764254.781041:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764254.781045:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764254.781047:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764254.781050:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764254.781051:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764254.781054:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000010:2.0:1787764254.781055:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff975202074000.
00000100:00000001:3.0:1787764254.781056:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764254.781059:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764254.781063:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832706816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764254.781066:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764254.781074:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832706432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764254.781075:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764254.781079:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764254.781087:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832706816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781088:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764254.781097:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832706432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764254.781098:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764254.781104:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21979)
00000100:00000001:3.0:1787764254.781107:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764254.781110:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764254.781112:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764254.781113:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764254.781114:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764254.781116:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764254.781118:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764254.781121:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764254.781123:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764254.781126:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832706432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764254.781131:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832706432:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764254.781153:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764254.781155:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764254.781157:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832706432/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781162:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764254.781164:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764254.781166:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764254.781168:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764254.781169:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764254.781170:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764254.781172:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764254.781174:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff975202077800.
02000000:00000001:3.0:1787764254.781177:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764254.781178:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764254.781179:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:3.0:1787764254.781181:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764254.781182:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764254.781184:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764254.781189:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764254.781190:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764254.781191:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764254.781192:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764254.781193:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764254.781196:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832706560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781200:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764254.781202:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764254.781203:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764254.781204:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764254.781206:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764254.781207:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832706560:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764254.781209:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764254.781211:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764254.781212:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764254.781213:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764254.781216:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764254.781217:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764254.781220:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764254.781222:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764254.781224:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764254.781227:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832706560, portal 10
00000100:00000001:3.0:1787764254.781228:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764254.781229:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:3.0:1787764254.781233:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832706560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781238:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764254.781239:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764254.781241:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764254.781242:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832706560, offset 0
00000400:00000200:3.0:1787764254.781245:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764254.781247:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764254.781250:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764254.781253:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764254.781255:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764254.781257:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764254.781258:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764254.781260:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58755:58755:256:4294967295] 192.168.203.110@tcp LPNI seq info [58755:58755:8:4294967295]
00000400:00000200:3.0:1787764254.781263:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764254.781267:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764254.781270:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764254.781273:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764254.781275:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764254.781277:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787764254.781280:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764254.781282:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1787764254.781285:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764254.781286:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764254.781288:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764254.781290:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764254.781292:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764254.781294:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764254.781296:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764254.781298:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832706560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781303:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764254.781305:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832706560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764279 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781310:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764254.781311:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764254.781313:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764254.781315:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764254.781317:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764254.781318:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316641824
00000100:00000001:3.0:1787764254.781321:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316643059
00000100:00000001:3.0:1787764254.781322:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316641824
00000100:00000001:3.0:1787764254.781324:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316643059
00000100:00000001:3.0:1787764254.781325:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764254.781327:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764254.781328:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764254.781330:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764254/real 1787764254]  req@ffff9751f86583c0 x1874599832706560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764254.781335:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764254.781335:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764254.781337:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764254.781338:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764254.781339:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764254.781341:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764254.781343:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832706560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764254.781348:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832706560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781352:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764254.781352:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764254.781353:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764254.781354:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764254.781356:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832706560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781360:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764254.781363:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832706560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781367:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764254.781369:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764254.781370:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764254.781371:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764254.781373:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764254.781382:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764254.781384:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764254.781386:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764254.781387:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764254.781390:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832706560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764254.781395:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832706560:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764254.781397:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764254.781399:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832706688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781403:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764254.781403:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764254.781405:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764254.781406:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764254.781407:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764254.781409:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832706688:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764254.781411:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764254.781412:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764254.781413:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764254.781414:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764254.781416:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764254.781418:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764254.781419:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764254.781421:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764254.781422:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764254.781424:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832706688, portal 10
00000100:00000001:3.0:1787764254.781425:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764254.781427:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:3.0:1787764254.781430:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832706688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781434:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764254.781436:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764254.781437:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764254.781438:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832706688, offset 0
00000400:00000200:3.0:1787764254.781441:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764254.781443:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764254.781445:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764254.781447:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764254.781449:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764254.781450:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764254.781452:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764254.781454:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58756:58756:256:4294967295] 192.168.203.110@tcp LPNI seq info [58756:58756:8:4294967295]
00000400:00000200:3.0:1787764254.781457:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764254.781461:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764254.781463:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764254.781480:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764254.781484:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764254.781486:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787764254.781488:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764254.781490:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1787764254.781493:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764254.781494:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764254.781496:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764254.781499:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764254.781501:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764254.781502:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764254.781504:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764254.781506:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832706688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781511:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764254.781514:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832706688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764279 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781518:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764254.781519:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764254.781521:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764254.781523:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764254.781526:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764254.781527:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316641812
00000100:00000001:3.0:1787764254.781529:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316643059
00000100:00000001:3.0:1787764254.781530:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316641812
00000100:00000001:3.0:1787764254.781532:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316643059
00000100:00000001:3.0:1787764254.781533:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764254.781535:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764254.781536:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764254.781538:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764254/real 1787764254]  req@ffff9751f8659a40 x1874599832706688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764254.781543:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764254.781544:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764254.781545:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764254.781546:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764254.781547:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764254.781548:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764254.781550:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832706688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764254.781555:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832706688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781559:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764254.781560:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764254.781561:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764254.781562:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764254.781565:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832706688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781569:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764254.781572:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832706688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781576:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764254.781577:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764254.781578:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764254.781579:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764254.781580:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764254.781581:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764254.781583:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764254.781584:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764254.781586:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764254.781588:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832706688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764254.781592:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832706688:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764254.781595:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764254.781596:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832706816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781600:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764254.781601:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764254.781603:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764254.781604:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764254.781605:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764254.781607:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832706816:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764254.781609:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764254.781611:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764254.781612:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764254.781613:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764254.781616:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764254.781617:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764254.781619:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764254.781621:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764254.781623:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764254.781625:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832706816, portal 4
00000100:00000001:3.0:1787764254.781626:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764254.781627:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:3.0:1787764254.781631:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832706816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781636:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764254.781637:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764254.781639:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764254.781640:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832706816, offset 0
00000400:00000200:3.0:1787764254.781642:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764254.781644:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764254.781647:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764254.781648:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764254.781650:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764254.781652:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764254.781653:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764254.781655:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58757:58757:256:4294967295] 192.168.203.110@tcp LPNI seq info [58757:58757:8:4294967295]
00000400:00000200:3.0:1787764254.781658:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764254.781662:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764254.781664:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764254.781666:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764254.781668:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764254.781670:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787764254.781673:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764254.781674:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1787764254.781677:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764254.781679:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764254.781681:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764254.781683:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764254.781685:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764254.781686:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764254.781688:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764254.781690:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832706816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781696:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764254.781697:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832706816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764279 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781702:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764254.781703:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764254.781705:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764254.781706:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764254.781708:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764254.781709:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316641829
00000100:00000001:3.0:1787764254.781711:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316643059
00000100:00000001:3.0:1787764254.781713:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316641829
00000100:00000001:3.0:1787764254.781715:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316643059
00000100:00000001:3.0:1787764254.781716:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764254.781717:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764254.781718:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764254.781720:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764254/real 1787764254]  req@ffff9751f8658000 x1874599832706816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764254.781725:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764254.781726:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764254.781726:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764254.781728:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764254.781729:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764254.781731:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764254.781733:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832706816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764254.781739:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832706816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781742:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764254.781743:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764254.781744:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764254.781745:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764254.781747:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832706816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781751:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764254.781753:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832706816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781757:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764254.781758:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764254.781759:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764254.781760:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764254.781761:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764254.781762:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764254.781764:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764254.781765:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764254.781766:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764254.781769:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832706816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764254.781773:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832706816:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764254.781776:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764254.781777:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764254.781779:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832706560/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781784:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764254.781785:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764254.781786:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764254.781787:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764254.781788:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764254.781789:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764254.781790:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764254.781791:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff975202074c00.
02000000:00000001:3.0:1787764254.781793:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764254.781794:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764254.781795:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:3.0:1787764254.781797:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764254.781798:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764254.781799:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764254.781801:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832706688/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781805:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764254.781806:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764254.781806:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764254.781808:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764254.781808:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764254.781809:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764254.781810:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764254.781811:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff975202077400.
02000000:00000001:3.0:1787764254.781812:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764254.781813:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764254.781814:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:3.0:1787764254.781815:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764254.781816:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764254.781817:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764254.781818:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832706816/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764279 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764254.781822:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764254.781823:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764254.781824:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764254.781826:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764254.781826:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764254.781827:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764254.781829:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764254.781829:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff975202074000.
02000000:00000001:3.0:1787764254.781830:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764254.781831:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764254.781832:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:3.0:1787764254.781833:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764254.781834:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764254.781835:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764254.781837:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764254.781838:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764254.781839:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764254.781840:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764255.420313:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764255.420315:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764255.420323:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764255.420324:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764255.420327:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764255.420328:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764255.420332:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764255.420334:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764255.420336:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764255.420339:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764255.420341:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764255.420345:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764255.420346:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764255.420348:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764255.420352:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764255.420353:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764255.420355:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764255.420364:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764255.420368:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764255.420373:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764255.420375:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764255.420434:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764255.420441:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764255.420446:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764255.804212:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764255.804219:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764255.804221:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764255.804231:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764255.804235:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764255.804238:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764255.804240:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764255.804242:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764255.804246:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764256.060244:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316643059, at 4316643072
00000100:00000010:3.0:1787764256.060258:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40c60.
00000100:00000040:2.0:1787764256.060269:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316643059, at 4316643072
00000100:00000010:2.0:1787764256.060277:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40360.
00000100:00000001:2.0:1787764256.444250:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764256.444252:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764256.444257:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764256.444260:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764256.444263:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764256.444264:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764256.444266:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764256.444272:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764256.444272:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764256.444274:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764256.444281:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764256.444286:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764256.444289:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764256.444292:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764256.444296:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764256.444322:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764256.444327:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764256.444331:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764256.828272:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764256.828280:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764256.828283:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764256.828288:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764256.828291:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764256.828295:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764256.828298:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764256.828300:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764256.828303:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764257.468242:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764257.468247:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764257.468251:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764257.468253:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764257.468255:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764257.468262:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764257.468264:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764257.468268:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764257.468270:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764257.468272:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764257.468274:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764257.468277:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764257.468280:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764257.468281:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764257.468286:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764257.468287:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764257.468291:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764257.468293:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764257.468295:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764257.468298:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764257.468299:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764257.468386:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764257.468394:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764257.468399:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764257.852257:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764257.852264:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764257.852266:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764257.852272:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764257.852275:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764257.852279:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764257.852282:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764257.852284:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764257.852287:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764258.492203:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764258.492215:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764258.492217:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764258.492222:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764258.492226:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764258.492226:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764258.492229:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764258.492233:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764258.492233:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764258.492236:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764258.492239:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764258.492242:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764258.492245:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764258.492250:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764258.492253:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764258.492291:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764258.492295:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764258.492298:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764258.876210:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764258.876217:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764258.876219:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764258.876224:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764258.876228:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764258.876232:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764258.876234:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764258.876236:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764258.876239:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764259.516225:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764259.516228:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764259.516233:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764259.516233:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764259.516236:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764259.516240:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764259.516242:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764259.516243:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764259.516248:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764259.516249:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764259.516251:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764259.516252:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764259.516256:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764259.516258:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764259.516261:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764259.516291:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764259.516296:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764259.516300:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764259.900237:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764259.900243:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764259.900246:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764259.900251:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764259.900255:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764259.900259:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.900261:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764259.900264:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.900267:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764259.900279:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764259.900288:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787764259.900292:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764259.900296:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764259.900302:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764259.900310:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21974
00000400:00000200:1.0:1787764259.900319:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764259.900325:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764259.900328:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764259.900331:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764259.900334:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764259.900336:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764259.900337:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764259.900341:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764259.900343:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787764259.900345:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764259.900349:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787764259.900350:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787764259.900352:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787764259.900355:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764259.900356:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764259.900358:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764259.900359:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764259.900361:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787764259.900364:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764259.900365:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764259.900367:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764259.900371:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764259.900374:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764259.900375:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764259.900378:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764259.900389:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933480.
00000020:00000040:1.0:1787764259.900392:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787764259.900396:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764259.900398:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764259.900401:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764259.900403:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764259.900408:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764259.900416:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb400.
00000100:00000001:1.0:1787764259.900423:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764259.900451:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933480 x1874599832706944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764259.900463:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764259.900469:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764259.900475:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3c2c33c] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933480 x1874599832706944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764259.900495:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764259.900498:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764259.900500:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787764259.900502:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764259.900503:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764259.900506:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764259.900509:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21974
00000400:00000200:1.0:1787764259.900513:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764259.900518:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764259.900520:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764259.900523:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764259.900524:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764259.900525:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764259.900528:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764259.900529:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764259.900531:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764259.900535:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764259.900537:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00000001:3.0:1787764259.900538:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:1.0:1787764259.900539:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764259.900546:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787764259.900551:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787764259.900556:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:3.0:1787764259.900558:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933480 x1874599832706944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787764259.900568:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764259.900578:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764259.900588:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764259.900590:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764259.900591:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764259.900593:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1787764259.900597:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:3.0:1787764259.900601:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:1.0:1787764259.900605:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:3.0:1787764259.900607:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764259.900613:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764259.900617:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764259.900618:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832706944:192.168.203.110@tcp:250:kworker.0
02000000:00000001:1.0:1787764259.900625:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764259.900627:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764259.900632:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764259.900633:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764259.900640:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764259.900640:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764259.900645:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1787764259.900645:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:1.0:1787764259.900654:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000010:3.0:1787764259.900657:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
00000008:00000001:1.0:1787764259.900659:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764259.900663:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764259.900665:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764259.900683:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930000.
00000400:00000200:3.0:1787764259.900688:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:1.0:1787764259.900694:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000010:3.0:1787764259.900695:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
02000000:00000001:1.0:1787764259.900698:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764259.900700:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764259.900702:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764259.900707:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764259.900709:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764259.900711:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
02000000:00000010:1.0:1787764259.900713:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9000.
00000100:00000001:1.0:1787764259.900720:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764259.900720:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832706944, portal 25
00000100:00000001:3.0:1787764259.900734:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00100000:1.0:1787764259.900739:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930000 x1874599832707072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764259.900739:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497734784 : -115097211816832 : ffff9751d7933480)
00000100:00000001:1.0:1787764259.900754:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:3.0:1787764259.900754:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933480 x1874599832706944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764259.900761:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764259.900778:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6d90274] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930000 x1874599832707072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.900779:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764259.900793:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764259.900801:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764259.900811:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832706944, offset 0
00000400:00000200:3.0:1787764259.900828:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787764259.900829:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764259.900834:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764259.900837:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:3.0:1787764259.900842:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764259.900844:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764259.900846:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764259.900848:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1787764259.900852:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00080000:1.0:1787764259.900856:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21974
00000400:00000200:3.0:1787764259.900864:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764259.900866:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764259.900872:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764259.900873:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764259.900879:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1787764259.900882:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000040:1.0:1787764259.900888:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:3.0:1787764259.900889:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:1.0:1787764259.900900:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764259.900913:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:3.0:1787764259.900914:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58758:58758:256:4294967295] 192.168.203.110@tcp LPNI seq info [58758:58758:8:4294967295]
00000100:00000001:1.0:1787764259.900926:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764259.900938:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764259.900943:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000400:00000200:3.0:1787764259.900943:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:1.0:1787764259.900949:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764259.900951:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787764259.900953:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787764259.900965:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:3.0:1787764259.900970:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:1.0:1787764259.900974:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764259.900982:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764259.900988:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000200:3.0:1787764259.900991:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787764259.900993:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764259.901000:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787764259.901002:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764259.901004:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1787764259.901005:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
02000000:00000001:1.0:1787764259.901007:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764259.901012:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764259.901014:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764259.901016:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764259.901018:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787764259.901024:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764259.901026:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764259.901029:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930b40.
00000020:00000040:1.0:1787764259.901031:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764259.901033:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000800:00000200:3.0:1787764259.901033:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:1.0:1787764259.901038:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787764259.901041:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:1.0:1787764259.901042:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764259.901043:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764259.901048:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787764259.901055:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000010:1.0:1787764259.901056:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca000.
00000800:00000200:3.1:1787764259.901064:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000100:00000001:1.0:1787764259.901065:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764259.901079:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930b40 x1874599832707200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000100:3.0:1787764259.901080:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764259.901086:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000100:00000001:1.0:1787764259.901098:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1787764259.901098:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:1.0:1787764259.901102:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764259.901111:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022d7f91a] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930b40 x1874599832707200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764259.901111:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764259.901123:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000100:00000001:1.0:1787764259.901127:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764259.901129:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000040:1.0:1787764259.901174:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764259.901176:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:1.0:1787764259.901178:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787764259.901181:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000200:3.0:1787764259.901184:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933480 x1874599832706944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787764259.901186:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:1.0:1787764259.901190:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764259.901194:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764259.901204:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764259.901213:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21974
00000100:00000001:3.0:1787764259.901217:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000400:00000200:1.0:1787764259.901221:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:3.0:1787764259.901226:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933480 x1874599832706944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764284 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787764259.901231:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764259.901238:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764259.901250:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764259.901259:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764259.901266:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764259.901268:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764259.901270:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764259.901273:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000001:1.0:1787764259.901277:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764259.901279:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787764259.901282:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787764259.901284:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1787764259.901285:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787764259.901292:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787764259.901300:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000200:3.0:1787764259.901300:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764259.901308:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316648180
00000100:00000040:1.0:1787764259.901311:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764259.901314:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000010:3.0:1787764259.901316:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40c60.
00000100:00000040:1.0:1787764259.901317:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764259.901320:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764259.901322:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:3.0:1787764259.901324:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00080000:1.0:1787764259.901326:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764259.901330:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764259.901332:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764259.901333:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316648180
02000000:00000001:1.0:1787764259.901335:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764259.901341:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764259.901342:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40780.
00000100:00000001:1.0:1787764259.901344:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764259.901352:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000008:00100000:1.0:1787764259.901353:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1787764259.901362:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764259.901362:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764259.901373:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764259.901373:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764259.901386:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000010:1.0:1787764259.901387:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7931680.
00000020:00000040:1.0:1787764259.901392:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787764259.901395:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764259.901396:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764259.901398:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764259.901401:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00100000:3.0:1787764259.901402:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764259/real 1787764259]  req@ffff9751d7933480 x1874599832706944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787764259.901403:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764259.901410:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb800.
00000100:00000001:1.0:1787764259.901419:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764259.901444:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7931680 x1874599832707328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000001:3.0:1787764259.901478:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:1.0:1787764259.901479:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000001:3.0:1787764259.901481:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:1.0:1787764259.901485:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764259.901486:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764259.901494:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000100:00000040:1.0:1787764259.901496:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d4255d6] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7931680 x1874599832707328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1787764259.901497:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764259.901506:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764259.901517:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933480 x1874599832706944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764259.901519:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764259.901533:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21984)
00000100:00000200:3.0:1787764259.901540:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933480 x1874599832706944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.901547:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764259.901548:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.901550:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764259.901552:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764259.901555:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933480 x1874599832706944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.901560:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764259.901564:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933480 x1874599832706944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.901568:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764259.901570:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.901571:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764259.901572:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.901574:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764259.901576:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764259.901579:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764259.901582:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764259.901583:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764259.901586:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933480 x1874599832706944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764259.901592:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832706944:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764259.901596:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.901597:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764259.901599:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933480 x1874599832706944/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.901603:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764259.901605:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764259.901607:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764259.901609:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764259.901611:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764259.901613:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764259.901614:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764259.901616:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb400.
02000000:00000001:3.0:1787764259.901618:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764259.901619:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764259.901621:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933480.
00000100:00000001:3.0:1787764259.901624:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764259.901625:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.901627:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.901633:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764259.901634:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764259.901635:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764259.901637:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764259.901638:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764259.901641:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930000 x1874599832707072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.901645:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764259.901646:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764259.901648:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764259.901650:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764259.901651:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764259.901653:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832707072:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764259.901656:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764259.901658:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764259.901659:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764259.901660:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764259.901663:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764259.901664:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764259.901667:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764259.901669:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764259.901672:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764259.901674:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832707072, portal 10
00000100:00000001:3.0:1787764259.901675:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764259.901676:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497721344 : -115097211830272 : ffff9751d7930000)
00000100:00000040:3.0:1787764259.901681:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930000 x1874599832707072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.901686:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764259.901687:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764259.901689:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764259.901690:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832707072, offset 0
00000400:00000200:3.0:1787764259.901693:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764259.901703:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764259.901706:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764259.901708:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764259.901713:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764259.901715:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764259.901716:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764259.901719:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58759:58759:256:4294967295] 192.168.203.110@tcp LPNI seq info [58759:58759:8:4294967295]
00000400:00000200:3.0:1787764259.901722:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764259.901726:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764259.901729:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764259.901731:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764259.901734:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764259.901737:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787764259.901741:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764259.901743:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1787764259.901745:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764259.901747:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764259.901749:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764259.901751:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764259.901753:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764259.901755:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764259.901756:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764259.901759:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930000 x1874599832707072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.901764:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764259.901766:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930000 x1874599832707072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764284 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.901770:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764259.901771:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764259.901773:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764259.901775:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764259.901777:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764259.901778:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316646944
00000100:00000001:3.0:1787764259.901781:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316648180
00000100:00000001:3.0:1787764259.901782:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316646944
00000100:00000001:3.0:1787764259.901784:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316648180
00000100:00000001:3.0:1787764259.901785:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764259.901786:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764259.901787:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764259.901790:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764259/real 1787764259]  req@ffff9751d7930000 x1874599832707072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764259.901794:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764259.901795:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764259.901797:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764259.901798:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764259.901799:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764259.901801:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764259.901803:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930000 x1874599832707072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764259.901808:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930000 x1874599832707072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.901812:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764259.901812:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.901813:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764259.901814:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764259.901816:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930000 x1874599832707072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.901820:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764259.901823:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930000 x1874599832707072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.901827:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764259.901829:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.901830:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764259.901831:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.901832:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764259.901842:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764259.901844:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764259.901846:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764259.901848:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764259.901850:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930000 x1874599832707072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764259.901855:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832707072:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764259.901858:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764259.901861:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930b40 x1874599832707200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.901865:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764259.901866:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764259.901868:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764259.901869:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764259.901870:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764259.901872:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832707200:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764259.901873:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764259.901875:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764259.901876:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764259.901878:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764259.901880:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764259.901881:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764259.901883:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764259.901885:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764259.901886:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764259.901888:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832707200, portal 10
00000100:00000001:3.0:1787764259.901890:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764259.901891:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497724224 : -115097211827392 : ffff9751d7930b40)
00000100:00000040:3.0:1787764259.901895:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930b40 x1874599832707200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.901899:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764259.901900:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764259.901902:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764259.901903:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832707200, offset 0
00000400:00000200:3.0:1787764259.901905:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764259.901907:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764259.901909:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764259.901911:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764259.901913:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764259.901915:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764259.901916:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764259.901919:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58760:58760:256:4294967295] 192.168.203.110@tcp LPNI seq info [58760:58760:8:4294967295]
00000400:00000200:3.0:1787764259.901921:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764259.901925:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764259.901928:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764259.901930:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764259.901932:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764259.901934:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787764259.901936:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764259.901939:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1787764259.901942:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764259.901943:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764259.901945:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764259.901948:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764259.901950:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764259.901951:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764259.901952:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764259.901955:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930b40 x1874599832707200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.901960:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764259.901962:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930b40 x1874599832707200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764284 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.901966:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764259.901967:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764259.901969:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764259.901970:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764259.901973:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764259.901974:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316646932
00000100:00000001:3.0:1787764259.901975:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316648180
00000100:00000001:3.0:1787764259.901977:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316646932
00000100:00000001:3.0:1787764259.901979:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316648180
00000100:00000001:3.0:1787764259.901981:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764259.901982:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764259.901984:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764259.901987:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764259/real 1787764259]  req@ffff9751d7930b40 x1874599832707200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764259.901996:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764259.901998:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764259.901999:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764259.902002:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764259.902004:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764259.902006:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764259.902009:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930b40 x1874599832707200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764259.902018:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930b40 x1874599832707200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.902025:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764259.902027:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.902029:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764259.902030:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764259.902033:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930b40 x1874599832707200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.902038:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764259.902040:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930b40 x1874599832707200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.902045:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764259.902046:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.902047:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764259.902048:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.902050:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764259.902051:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764259.902053:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764259.902055:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764259.902057:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764259.902059:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930b40 x1874599832707200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764259.902064:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832707200:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764259.902066:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764259.902068:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7931680 x1874599832707328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.902072:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764259.902073:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764259.902075:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764259.902076:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764259.902078:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764259.902080:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832707328:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764259.902082:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764259.902083:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764259.902084:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764259.902085:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764259.902088:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764259.902090:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764259.902092:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764259.902094:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764259.902096:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764259.902098:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832707328, portal 4
00000100:00000001:3.0:1787764259.902099:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764259.902100:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497727104 : -115097211824512 : ffff9751d7931680)
00000100:00000040:3.0:1787764259.902104:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7931680 x1874599832707328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.902109:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764259.902111:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764259.902112:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764259.902114:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832707328, offset 0
00000400:00000200:3.0:1787764259.902116:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764259.902119:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764259.902121:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764259.902123:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764259.902125:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764259.902127:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764259.902128:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764259.902130:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58761:58761:256:4294967295] 192.168.203.110@tcp LPNI seq info [58761:58761:8:4294967295]
00000400:00000200:3.0:1787764259.902150:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764259.902155:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764259.902158:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764259.902160:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764259.902163:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764259.902165:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787764259.902168:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764259.902170:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1787764259.902173:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764259.902174:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764259.902176:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764259.902179:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764259.902181:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764259.902182:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764259.902184:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764259.902186:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7931680 x1874599832707328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.902192:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764259.902194:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7931680 x1874599832707328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764284 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.902202:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764259.902203:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764259.902205:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764259.902208:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764259.902210:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764259.902211:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316646950
00000100:00000001:3.0:1787764259.902213:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316648180
00000100:00000001:3.0:1787764259.902214:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316646950
00000100:00000001:3.0:1787764259.902216:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316648180
00000100:00000001:3.0:1787764259.902217:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764259.902218:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764259.902220:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764259.902222:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764259/real 1787764259]  req@ffff9751d7931680 x1874599832707328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764259.902226:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764259.902227:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764259.902228:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764259.902230:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764259.902231:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764259.902232:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764259.902234:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7931680 x1874599832707328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764259.902239:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7931680 x1874599832707328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.902244:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764259.902244:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.902245:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764259.902246:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764259.902248:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7931680 x1874599832707328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.902253:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764259.902255:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7931680 x1874599832707328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.902260:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764259.902261:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.902262:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764259.902263:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.902264:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764259.902265:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764259.902266:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764259.902268:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764259.902269:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764259.902272:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7931680 x1874599832707328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764259.902276:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832707328:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764259.902279:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.902280:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764259.902282:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930000 x1874599832707072/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.902286:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764259.902287:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764259.902288:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764259.902290:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764259.902291:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764259.902292:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764259.902293:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764259.902295:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9000.
02000000:00000001:3.0:1787764259.902297:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764259.902298:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764259.902299:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930000.
00000100:00000001:3.0:1787764259.902301:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764259.902302:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.902303:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764259.902305:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930b40 x1874599832707200/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.902309:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764259.902310:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764259.902311:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764259.902312:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764259.902312:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764259.902313:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764259.902314:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764259.902315:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca000.
02000000:00000001:3.0:1787764259.902316:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764259.902317:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764259.902318:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930b40.
00000100:00000001:3.0:1787764259.902319:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764259.902320:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.902321:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764259.902322:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7931680 x1874599832707328/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764284 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764259.902326:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764259.902327:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764259.902328:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764259.902329:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764259.902330:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764259.902330:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764259.902331:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764259.902332:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb800.
02000000:00000001:3.0:1787764259.902333:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764259.902334:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764259.902335:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7931680.
00000100:00000001:3.0:1787764259.902337:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764259.902338:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.902339:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764259.902341:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764259.902342:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764259.902344:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764259.902345:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764260.540218:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764260.540221:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764260.540225:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764260.540226:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764260.540230:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764260.540232:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764260.540235:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764260.540235:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764260.540240:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764260.540241:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764260.540244:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764260.540245:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764260.540248:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764260.540251:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764260.540253:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764260.540293:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764260.540299:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764260.540303:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764260.924246:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764260.924252:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764260.924254:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764260.924259:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764260.924262:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764260.924265:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764260.924268:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764260.924270:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764260.924272:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764261.180222:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316648180, at 4316648192
00000100:00000010:3.0:1787764261.180235:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40780.
00000100:00000040:2.0:1787764261.180256:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316648180, at 4316648192
00000100:00000010:2.0:1787764261.180265:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40c60.
00000100:00000001:2.0:1787764261.564219:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764261.564227:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764261.564229:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764261.564234:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764261.564238:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764261.564240:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764261.564243:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764261.564245:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764261.564248:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764261.564255:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764261.564262:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764261.564269:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764261.564271:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764261.564275:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764261.564277:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764261.564321:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764261.564326:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764261.564328:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764261.948257:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764261.948265:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764261.948267:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764261.948273:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764261.948278:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764261.948281:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764261.948285:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764261.948288:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764261.948291:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764262.588250:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764262.588252:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764262.588261:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764262.588263:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764262.588268:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764262.588270:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764262.588277:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764262.588277:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764262.588281:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764262.588282:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764262.588288:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764262.588291:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764262.588294:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764262.588295:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764262.588302:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764262.588307:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764262.588311:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764262.588315:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764262.588318:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764262.588320:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764262.588324:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764262.588366:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764262.588370:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764262.588373:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764262.972268:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764262.972278:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764262.972280:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764262.972287:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764262.972291:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764262.972296:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764262.972298:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764262.972301:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764262.972305:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764263.612223:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764263.612224:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764263.612232:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764263.612234:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764263.612238:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764263.612239:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764263.612243:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764263.612244:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764263.612247:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764263.612249:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764263.612251:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764263.612255:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764263.612260:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764263.612263:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764263.612266:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764263.612297:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764263.612301:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764263.612303:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764263.996237:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764263.996247:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764263.996250:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764263.996256:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764263.996261:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764263.996270:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764263.996274:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764263.996276:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764263.996280:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764264.636257:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764264.636266:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764264.636266:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764264.636274:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764264.636276:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764264.636278:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764264.636281:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764264.636286:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764264.636286:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764264.636291:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764264.636298:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764264.636302:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764264.636305:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764264.636307:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764264.636311:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764264.636341:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764264.636345:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764264.636348:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:1.0:1787764265.020311:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764265.020325:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:1.0:1787764265.020326:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787764265.020330:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764265.020339:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764265.020341:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764265.020346:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764265.020355:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764265.020361:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764265.020368:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764265.020383:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764265.020384:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764265.020407:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.020416:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764265.020423:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21979
00000400:00000200:1.0:1787764265.020438:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764265.020454:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764265.020461:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764265.020469:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764265.020475:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764265.020481:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764265.020488:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764265.020494:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764265.020497:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787764265.020501:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764265.020506:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787764265.020510:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787764265.020514:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787764265.020520:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764265.020523:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764265.020526:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764265.020529:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764265.020535:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787764265.020541:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764265.020545:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764265.020549:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764265.020554:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764265.020558:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764265.020562:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764265.020566:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764265.020579:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7931680.
00000020:00000040:1.0:1787764265.020585:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787764265.020590:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764265.020593:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764265.020597:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764265.020600:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764265.020606:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764265.020619:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca800.
00000100:00000001:1.0:1787764265.020628:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764265.020641:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7931680 x1874599832707456/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764265.020656:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764265.020663:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764265.020671:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d4255d6] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7931680 x1874599832707456/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764265.020701:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764265.020706:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764265.020712:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787764265.020714:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764265.020716:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764265.020721:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764265.020725:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21979
00000400:00000200:1.0:1787764265.020731:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764265.020736:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764265.020740:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764265.020744:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764265.020748:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764265.020751:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764265.020754:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764265.020758:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764265.020760:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787764265.020763:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764265.020766:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787764265.020771:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787764265.020775:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787764265.020781:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764265.020784:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764265.020787:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764265.020791:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764265.020794:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787764265.020798:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764265.020818:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764265.020823:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764265.020831:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764265.020835:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764265.020838:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764265.020845:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787764265.020850:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764265.020853:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764265.020860:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930b40.
00000020:00000040:1.0:1787764265.020864:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764265.020867:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764265.020868:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764265.020871:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764265.020873:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764265.020876:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764265.020884:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8400.
00000100:00000001:1.0:1787764265.020891:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764265.020901:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930b40 x1874599832707584/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764265.020913:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764265.020926:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764265.020935:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022d7f91a] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930b40 x1874599832707584/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764265.020948:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764265.020952:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764265.020958:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787764265.020962:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764265.020966:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764265.020972:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764265.020976:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21979
00000100:00000001:3.0:1787764265.020982:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:1.0:1787764265.020989:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764265.020994:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000400:00000200:1.0:1787764265.020997:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787764265.021002:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00080000:1.0:1787764265.021004:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764265.021013:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764265.021017:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764265.021026:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7931680 x1874599832707456/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787764265.021031:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764265.021046:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764265.021056:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764265.021063:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000100:00000001:3.0:1787764265.021066:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:1.0:1787764265.021070:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787764265.021072:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1787764265.021078:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
02000000:00000001:3.0:1787764265.021083:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:1.0:1787764265.021086:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
02000000:00000001:3.0:1787764265.021092:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:1.0:1787764265.021094:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
02000000:00000001:3.0:1787764265.021103:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764265.021111:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00100000:3.0:1787764265.021118:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832707456:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787764265.021122:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764265.021178:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764265.021186:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764265.021193:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764265.021194:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:3.0:1787764265.021203:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1787764265.021212:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000010:3.0:1787764265.021213:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764265.021220:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1787764265.021224:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787764265.021236:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:3.0:1787764265.021238:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764265.021248:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000100:00000001:1.0:1787764265.021250:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764265.021258:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764265.021264:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:3.0:1787764265.021264:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
02000000:00000001:1.0:1787764265.021274:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764265.021278:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000200:3.0:1787764265.021281:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832707456, portal 25
00000008:00100000:1.0:1787764265.021282:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787764265.021290:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000008:00000001:1.0:1787764265.021293:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764265.021297:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497727104 : -115097211824512 : ffff9751d7931680)
00000100:00000001:1.0:1787764265.021299:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764265.021312:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930000.
00000100:00000040:3.0:1787764265.021316:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7931680 x1874599832707456/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764290 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787764265.021319:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764265.021331:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764265.021340:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764265.021352:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764265.021357:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1787764265.021357:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:1.0:1787764265.021364:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764265.021371:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
02000000:00000010:1.0:1787764265.021377:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8800.
00000400:00000010:3.0:1787764265.021379:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764265.021389:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832707456, offset 0
00000100:00000001:1.0:1787764265.021390:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764265.021418:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00100000:1.0:1787764265.021429:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930000 x1874599832707712/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787764265.021442:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764265.021454:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764265.021459:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764265.021462:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764265.021475:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:1.0:1787764265.021476:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6d90274] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930000 x1874599832707712/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764265.021485:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764265.021499:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1787764265.021509:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764265.021509:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764265.021519:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58762:58762:256:4294967295] 192.168.203.110@tcp LPNI seq info [58762:58762:8:4294967295]
00000100:00000040:1.0:1787764265.021521:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1787764265.021533:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:1.0:1787764265.021542:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787764265.021549:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1787764265.021554:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764265.021562:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764265.021569:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000100:00080000:1.0:1787764265.021574:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000800:00000200:3.0:1787764265.021580:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787764265.021587:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:3.0:1787764265.021589:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000100:00080000:1.0:1787764265.021595:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:3.1:1787764265.021598:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764265.021607:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000100:00000001:1.0:1787764265.021608:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000800:00000100:3.0:1787764265.021615:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764265.021622:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000100:00080000:1.0:1787764265.021623:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21979
00000400:00000200:3.0:1787764265.021630:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764265.021638:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764265.021646:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764265.021654:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000200:1.0:1787764265.021659:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000010:3.0:1787764265.021660:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00080000:1.0:1787764265.021671:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787764265.021672:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764265.021684:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7931680 x1874599832707456/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764290 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787764265.021689:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764265.021694:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764265.021696:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764265.021706:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764265.021719:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764265.021722:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787764265.021725:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787764265.021726:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000040:1.0:1787764265.021729:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000040:3.0:1787764265.021736:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7931680 x1874599832707456/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764290 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1787764265.021737:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787764265.021753:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:3.0:1787764265.021769:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764265.021770:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764265.021780:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764265.021780:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:1.0:1787764265.021793:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764265.021795:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764265.021797:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764265.021806:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000400:00000200:3.0:1787764265.021814:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:1.0:1787764265.021821:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764265.021823:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316653300
00000100:00000001:1.0:1787764265.021837:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764265.021840:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40d20.
02000000:00000001:1.0:1787764265.021842:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764265.021850:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764265.021853:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
02000000:00000001:1.0:1787764265.021854:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764265.021863:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764265.021869:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1787764265.021872:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316653300
00000008:00000001:1.0:1787764265.021873:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764265.021879:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764265.021879:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40ae0.
00000100:00000010:1.0:1787764265.021895:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933480.
00000100:00000001:3.0:1787764265.021896:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000020:00000040:1.0:1787764265.021910:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:3.0:1787764265.021913:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764265.021914:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764265.021917:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764265.021923:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764265.021926:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1787764265.021926:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764265.021931:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764265.021936:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000010:1.0:1787764265.021938:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8c00.
00000100:00000001:1.0:1787764265.021945:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764265.021952:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764265/real 1787764265]  req@ffff9751d7931680 x1874599832707456/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764265.021956:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933480 x1874599832707840/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764265.022037:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000001:3.0:1787764265.022039:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:1.0:1787764265.022043:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1787764265.022044:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764265.022053:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000100:00000040:1.0:1787764265.022058:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3c2c33c] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933480 x1874599832707840/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764265.022060:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764265.022067:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764265.022077:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764265.022097:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7931680 x1874599832707456/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764265.022111:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764265.022119:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7931680 x1874599832707456/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787764265.022123:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21989)
00000100:00000001:3.0:1787764265.022127:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764265.022179:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.022184:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764265.022186:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764265.022193:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7931680 x1874599832707456/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022201:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764265.022205:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7931680 x1874599832707456/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022211:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764265.022215:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.022217:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764265.022219:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.022222:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764265.022224:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764265.022227:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764265.022231:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764265.022232:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764265.022236:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7931680 x1874599832707456/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764265.022256:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832707456:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764265.022262:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764265.022266:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930b40 x1874599832707584/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022274:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764265.022277:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764265.022280:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764265.022282:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764265.022285:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764265.022289:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832707584:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764265.022293:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764265.022296:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764265.022298:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764265.022301:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764265.022305:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764265.022307:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764265.022312:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764265.022314:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764265.022318:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764265.022321:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832707584, portal 10
00000100:00000001:3.0:1787764265.022322:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764265.022323:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497724224 : -115097211827392 : ffff9751d7930b40)
00000100:00000040:3.0:1787764265.022330:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930b40 x1874599832707584/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764290 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022334:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764265.022336:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764265.022338:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764265.022340:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832707584, offset 0
00000400:00000200:3.0:1787764265.022342:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764265.022345:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764265.022348:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764265.022351:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764265.022353:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764265.022354:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764265.022356:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764265.022359:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58763:58763:256:4294967295] 192.168.203.110@tcp LPNI seq info [58763:58763:8:4294967295]
00000400:00000200:3.0:1787764265.022361:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764265.022367:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764265.022370:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764265.022372:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764265.022376:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764265.022378:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787764265.022381:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764265.022383:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787764265.022386:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764265.022388:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764265.022391:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764265.022393:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764265.022411:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764265.022413:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764265.022415:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764265.022418:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930b40 x1874599832707584/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764290 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022424:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764265.022426:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930b40 x1874599832707584/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764290 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022431:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764265.022432:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764265.022435:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764265.022438:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764265.022441:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764265.022442:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316652065
00000100:00000001:3.0:1787764265.022445:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316653300
00000100:00000001:3.0:1787764265.022447:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316652065
00000100:00000001:3.0:1787764265.022449:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316653300
00000100:00000001:3.0:1787764265.022450:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764265.022451:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764265.022453:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764265.022455:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764265/real 1787764265]  req@ffff9751d7930b40 x1874599832707584/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764265.022461:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764265.022462:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764265.022463:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764265.022464:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764265.022465:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764265.022467:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764265.022469:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930b40 x1874599832707584/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764265.022475:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930b40 x1874599832707584/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022479:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764265.022480:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.022482:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764265.022482:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764265.022485:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930b40 x1874599832707584/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022489:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764265.022491:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930b40 x1874599832707584/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022495:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764265.022497:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.022498:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764265.022498:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.022500:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764265.022511:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764265.022512:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764265.022514:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764265.022515:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764265.022518:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930b40 x1874599832707584/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764265.022523:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832707584:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764265.022525:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.022528:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764265.022530:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7931680 x1874599832707456/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022535:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764265.022538:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764265.022540:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764265.022542:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764265.022545:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764265.022546:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764265.022549:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764265.022551:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca800.
02000000:00000001:3.0:1787764265.022555:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764265.022557:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764265.022558:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7931680.
00000100:00000001:3.0:1787764265.022562:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764265.022563:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.022564:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764265.022566:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930b40 x1874599832707584/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022570:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764265.022571:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764265.022572:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764265.022573:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764265.022574:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764265.022575:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764265.022576:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764265.022577:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8400.
02000000:00000001:3.0:1787764265.022578:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764265.022579:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764265.022580:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930b40.
00000100:00000001:3.0:1787764265.022581:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764265.022582:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.022583:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.022591:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764265.022592:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764265.022593:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764265.022595:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764265.022596:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764265.022599:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930000 x1874599832707712/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022604:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764265.022605:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764265.022607:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764265.022608:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764265.022609:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764265.022611:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832707712:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764265.022614:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764265.022616:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764265.022617:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764265.022619:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764265.022621:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764265.022622:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764265.022624:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764265.022626:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764265.022628:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764265.022631:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832707712, portal 10
00000100:00000001:3.0:1787764265.022632:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764265.022633:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497721344 : -115097211830272 : ffff9751d7930000)
00000100:00000040:3.0:1787764265.022638:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930000 x1874599832707712/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764290 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022642:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764265.022644:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764265.022646:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764265.022647:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832707712, offset 0
00000400:00000200:3.0:1787764265.022650:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764265.022652:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764265.022655:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764265.022657:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764265.022658:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764265.022660:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764265.022662:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764265.022664:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58764:58764:256:4294967295] 192.168.203.110@tcp LPNI seq info [58764:58764:8:4294967295]
00000400:00000200:3.0:1787764265.022668:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764265.022672:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764265.022675:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764265.022677:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764265.022680:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764265.022683:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787764265.022686:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764265.022688:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787764265.022690:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764265.022692:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764265.022694:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764265.022696:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764265.022699:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764265.022700:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764265.022702:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764265.022704:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930000 x1874599832707712/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764290 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022709:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764265.022712:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930000 x1874599832707712/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764290 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022716:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764265.022717:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764265.022718:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764265.022720:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764265.022722:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764265.022724:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316652053
00000100:00000001:3.0:1787764265.022726:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316653300
00000100:00000001:3.0:1787764265.022727:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316652053
00000100:00000001:3.0:1787764265.022729:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316653300
00000100:00000001:3.0:1787764265.022732:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764265.022733:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764265.022734:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764265.022737:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764265/real 1787764265]  req@ffff9751d7930000 x1874599832707712/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764265.022742:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764265.022743:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764265.022744:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764265.022745:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764265.022746:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764265.022747:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764265.022749:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930000 x1874599832707712/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764265.022754:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930000 x1874599832707712/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022758:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764265.022759:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.022761:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764265.022762:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764265.022764:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930000 x1874599832707712/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022768:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764265.022770:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930000 x1874599832707712/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022774:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764265.022775:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.022776:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764265.022777:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.022778:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764265.022779:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764265.022782:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764265.022784:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764265.022786:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764265.022788:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930000 x1874599832707712/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764265.022793:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832707712:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764265.022795:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764265.022798:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933480 x1874599832707840/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022802:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764265.022803:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764265.022804:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764265.022805:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764265.022806:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764265.022808:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832707840:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764265.022810:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764265.022811:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764265.022812:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764265.022813:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764265.022815:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764265.022816:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764265.022818:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764265.022820:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764265.022822:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764265.022824:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832707840, portal 4
00000100:00000001:3.0:1787764265.022825:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764265.022826:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497734784 : -115097211816832 : ffff9751d7933480)
00000100:00000040:3.0:1787764265.022830:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933480 x1874599832707840/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764290 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022839:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764265.022841:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764265.022842:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764265.022844:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832707840, offset 0
00000400:00000200:3.0:1787764265.022846:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764265.022848:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764265.022850:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764265.022851:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764265.022853:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764265.022854:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764265.022856:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764265.022858:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58765:58765:256:4294967295] 192.168.203.110@tcp LPNI seq info [58765:58765:8:4294967295]
00000400:00000200:3.0:1787764265.022861:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764265.022865:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764265.022868:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764265.022870:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764265.022872:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764265.022874:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787764265.022876:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764265.022878:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787764265.022880:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764265.022881:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764265.022883:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764265.022885:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764265.022887:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764265.022889:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764265.022891:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764265.022894:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933480 x1874599832707840/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764290 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022902:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764265.022906:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933480 x1874599832707840/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764290 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022920:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764265.022921:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764265.022925:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764265.022928:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764265.022932:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764265.022935:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316652070
00000100:00000001:3.0:1787764265.022937:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316653300
00000100:00000001:3.0:1787764265.022940:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316652070
00000100:00000001:3.0:1787764265.022943:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316653300
00000100:00000001:3.0:1787764265.022945:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764265.022947:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764265.022949:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764265.022953:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764265/real 1787764265]  req@ffff9751d7933480 x1874599832707840/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764265.022960:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764265.022961:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764265.022963:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764265.022964:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764265.022966:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764265.022967:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764265.022970:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933480 x1874599832707840/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764265.022975:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933480 x1874599832707840/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022979:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764265.022980:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.022981:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764265.022982:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764265.022984:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933480 x1874599832707840/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022989:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764265.022992:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933480 x1874599832707840/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.022996:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764265.022998:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.022999:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764265.022999:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.023001:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764265.023002:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764265.023003:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764265.023005:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764265.023006:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764265.023009:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933480 x1874599832707840/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764265.023013:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832707840:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764265.023015:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.023017:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764265.023019:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930000 x1874599832707712/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.023023:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764265.023024:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764265.023025:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764265.023027:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764265.023029:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764265.023030:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764265.023031:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764265.023032:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8800.
02000000:00000001:3.0:1787764265.023034:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764265.023036:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764265.023037:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930000.
00000100:00000001:3.0:1787764265.023038:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764265.023039:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.023040:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764265.023043:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933480 x1874599832707840/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764290 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764265.023047:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764265.023048:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764265.023049:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764265.023050:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764265.023050:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764265.023051:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764265.023052:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764265.023053:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8c00.
02000000:00000001:3.0:1787764265.023054:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764265.023055:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764265.023056:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933480.
00000100:00000001:3.0:1787764265.023059:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764265.023060:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.023061:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764265.023063:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764265.023064:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764265.023065:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764265.023066:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764265.660237:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764265.660241:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764265.660244:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764265.660247:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764265.660247:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764265.660251:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00100000:2.0:1787764265.660253:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764265.660255:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764265.660257:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764265.660260:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764265.660261:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764265.660263:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764265.660265:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764265.660266:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764265.660268:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764265.660270:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764265.660272:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764265.660275:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764265.660277:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764265.660280:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764265.660282:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764265.660342:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764265.660347:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764265.660350:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764266.044297:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764266.044309:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764266.044312:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764266.044320:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764266.044326:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764266.044331:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764266.044335:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764266.044338:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764266.044342:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764266.300228:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316653300, at 4316653312
00000100:00000010:3.0:1787764266.300261:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40ae0.
00000100:00000040:2.0:1787764266.300270:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316653300, at 4316653312
00000100:00000010:2.0:1787764266.300278:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40d20.
00000100:00000001:2.0:1787764266.684251:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764266.684256:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764266.684265:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764266.684273:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764266.684275:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764266.684279:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764266.684281:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764266.684284:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764266.684288:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764266.684291:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764266.684295:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764266.684299:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764266.684302:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764266.684305:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764266.684308:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764266.684336:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764266.684342:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764266.684345:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764267.068282:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764267.068292:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764267.068294:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764267.068299:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764267.068303:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764267.068307:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764267.068309:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764267.068311:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764267.068314:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764267.708256:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764267.708261:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764267.708266:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764267.708268:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764267.708270:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764267.708275:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764267.708280:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764267.708280:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764267.708286:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764267.708287:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764267.708289:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764267.708291:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764267.708293:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764267.708296:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764267.708297:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764267.708300:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764267.708303:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764267.708305:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764267.708307:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764267.708313:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764267.708314:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764267.708390:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764267.708396:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764267.708400:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764268.092275:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764268.092287:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764268.092290:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764268.092297:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764268.092301:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764268.092305:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764268.092307:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764268.092311:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764268.092314:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764268.732229:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764268.732233:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764268.732236:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764268.732238:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764268.732241:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764268.732242:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764268.732244:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764268.732244:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764268.732247:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764268.732248:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764268.732249:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764268.732252:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764268.732255:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764268.732258:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764268.732261:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764268.732290:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764268.732294:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764268.732297:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764269.116243:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764269.116251:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764269.116254:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764269.116259:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764269.116263:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764269.116266:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764269.116268:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764269.116271:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764269.116274:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764269.756221:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764269.756225:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764269.756228:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764269.756231:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764269.756231:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764269.756238:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764269.756239:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764269.756243:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764269.756244:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764269.756248:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764269.756248:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764269.756252:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764269.756253:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764269.756256:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764269.756259:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764269.756303:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764269.756306:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764269.756309:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764270.140198:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764270.140206:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764270.140208:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764270.140212:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764270.140216:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:1.0:1787764270.140219:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764270.140220:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764270.140224:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764270.140226:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764270.140230:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764270.140231:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787764270.140235:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764270.140240:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764270.140245:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764270.140253:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21984
00000400:00000200:1.0:1787764270.140260:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764270.140266:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764270.140269:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764270.140272:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764270.140275:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764270.140277:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764270.140279:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764270.140283:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764270.140285:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787764270.140287:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764270.140289:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787764270.140291:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787764270.140293:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787764270.140295:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764270.140296:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764270.140297:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764270.140299:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764270.140301:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787764270.140304:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764270.140306:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764270.140308:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764270.140311:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764270.140313:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764270.140315:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764270.140317:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764270.140323:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932940.
00000020:00000040:1.0:1787764270.140326:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787764270.140328:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764270.140330:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764270.140332:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764270.140334:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764270.140337:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764270.140344:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca400.
00000100:00000001:1.0:1787764270.140349:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764270.140356:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932940 x1874599832707968/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764270.140375:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764270.140379:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764270.140383:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000af90eaa] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932940 x1874599832707968/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764270.140397:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764270.140400:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764270.140402:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787764270.140403:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764270.140405:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764270.140407:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764270.140410:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21984
00000400:00000200:1.0:1787764270.140413:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764270.140415:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764270.140417:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764270.140420:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764270.140422:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764270.140424:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764270.140424:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764270.140428:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764270.140429:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:1.0:1787764270.140432:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1787764270.140433:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:1.0:1787764270.140434:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787764270.140437:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764270.140438:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000040:3.0:1787764270.140441:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932940 x1874599832707968/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1787764270.140442:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787764270.140446:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787764270.140452:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764270.140456:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764270.140460:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764270.140460:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764270.140464:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764270.140464:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1787764270.140469:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:3.0:1787764270.140469:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764270.140473:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00080000:1.0:1787764270.140474:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:3.0:1787764270.140479:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764270.140480:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764270.140483:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:3.0:1787764270.140485:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832707968:192.168.203.110@tcp:250:kworker.0
02000000:00000001:1.0:1787764270.140490:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764270.140491:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764270.140494:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764270.140495:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1787764270.140496:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1787764270.140499:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:1.0:1787764270.140502:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1787764270.140502:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:1.0:1787764270.140507:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764270.140508:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764270.140509:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764270.140512:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764270.140513:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933840.
00000020:00000040:1.0:1787764270.140516:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000200:3.0:1787764270.140520:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:1.0:1787764270.140521:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764270.140523:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764270.140525:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:3.0:1787764270.140525:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
02000000:00000001:1.0:1787764270.140529:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764270.140536:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764270.140539:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
02000000:00000010:1.0:1787764270.140542:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8c00.
00000100:00000001:1.0:1787764270.140547:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764270.140547:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832707968, portal 25
00000100:00000001:3.0:1787764270.140552:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764270.140554:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497731904 : -115097211819712 : ffff9751d7932940)
00000100:00100000:1.0:1787764270.140557:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933840 x1874599832708096/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:3.0:1787764270.140563:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932940 x1874599832707968/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764295 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764270.140568:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764270.140572:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764270.140578:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764270.140580:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000083dd6d32] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933840 x1874599832708096/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787764270.140585:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764270.140590:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000001:1.0:1787764270.140592:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764270.140594:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832707968, offset 0
00000100:00080000:1.0:1787764270.140597:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764270.140602:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:3.0:1787764270.140602:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787764270.140604:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764270.140606:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764270.140608:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764270.140612:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21984
00000400:00000200:3.0:1787764270.140612:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764270.140618:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764270.140622:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764270.140623:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764270.140628:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1787764270.140630:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:1.0:1787764270.140632:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764270.140636:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787764270.140637:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000040:1.0:1787764270.140641:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:3.0:1787764270.140644:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:1.0:1787764270.140645:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:3.0:1787764270.140648:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000020:00000001:1.0:1787764270.140650:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764270.140652:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000400:00000200:3.0:1787764270.140655:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58766:58766:256:4294967295] 192.168.203.110@tcp LPNI seq info [58766:58766:8:4294967295]
00000020:00000001:1.0:1787764270.140656:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764270.140659:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000400:00000200:3.0:1787764270.140663:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:1.0:1787764270.140664:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787764270.140668:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787764270.140672:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764270.140676:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787764270.140679:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:1.0:1787764270.140682:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764270.140687:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764270.140691:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000800:00000200:3.0:1787764270.140692:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764270.140698:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000100:00080000:1.0:1787764270.140702:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000800:00000200:3.0:1787764270.140707:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787764270.140708:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764270.140712:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:3.0:1787764270.140713:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
02000000:00000001:1.0:1787764270.140715:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764270.140718:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764270.140722:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:3.1:1787764270.140722:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000008:00100000:1.0:1787764270.140725:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000800:00000200:3.1:1787764270.140727:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000008:00000001:1.0:1787764270.140728:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764270.140733:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1787764270.140736:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000010:1.0:1787764270.140739:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932580.
00000800:00000010:3.0:1787764270.140740:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000020:00000040:1.0:1787764270.140743:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764270.140747:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:3.0:1787764270.140747:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:1.0:1787764270.140750:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764270.140752:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1787764270.140755:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:1.0:1787764270.140756:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764270.140759:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764270.140762:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
02000000:00000010:1.0:1787764270.140763:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8800.
00000400:00000010:3.0:1787764270.140764:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:1.0:1787764270.140768:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764270.140770:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00100000:1.0:1787764270.140774:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932580 x1874599832708224/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1787764270.140776:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932940 x1874599832707968/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764295 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764270.140800:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764270.140802:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:1.0:1787764270.140807:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764270.140808:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932940 x1874599832707968/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764295 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.140818:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764270.140819:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d026924] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932580 x1874599832708224/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.140820:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764270.140825:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764270.140833:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764270.140837:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316658419
00000100:00000001:1.0:1787764270.140844:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764270.140845:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40ae0.
00000100:00000040:1.0:1787764270.140849:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764270.140853:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000040:1.0:1787764270.140854:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787764270.140859:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764270.140862:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787764270.140862:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316658419
00000100:00000001:1.0:1787764270.140865:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764270.140866:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764270.140868:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764270.140870:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21984
00000400:00000200:1.0:1787764270.140875:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000010:3.0:1787764270.140876:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40ba0.
00000400:00000200:1.0:1787764270.140879:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787764270.140881:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00080000:1.0:1787764270.140882:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787764270.140887:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:1.0:1787764270.140888:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764270.140893:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764270.140893:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764270.140898:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764270.140898:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:1.0:1787764270.140902:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00100000:3.0:1787764270.140905:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764270/real 1787764270]  req@ffff9751d7932940 x1874599832707968/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:1.0:1787764270.140906:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764270.140908:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787764270.140911:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764270.140914:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787764270.140919:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000001:3.0:1787764270.140921:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000001:1.0:1787764270.140922:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000001:3.0:1787764270.140924:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764270.140927:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000100:00000040:1.0:1787764270.140928:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764270.140932:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787764270.140932:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764270.140937:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000040:1.0:1787764270.140938:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764270.140941:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:1.0:1787764270.140942:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764270.140945:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000200:3.0:1787764270.140946:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932940 x1874599832707968/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:1.0:1787764270.140950:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764270.140956:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764270.140960:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764270.140964:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:3.0:1787764270.140964:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932940 x1874599832707968/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787764270.140970:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764270.140975:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764270.140979:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1787764270.140979:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764270.140981:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000008:00000001:1.0:1787764270.140986:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764270.140987:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764270.140990:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764270.140990:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:1.0:1787764270.140996:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930780.
00000100:00100000:3.0:1787764270.140996:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932940 x1874599832707968/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:1.0:1787764270.141001:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787764270.141006:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764270.141009:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764270.141011:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764270.141012:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764270.141014:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764270.141017:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764270.141018:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932940 x1874599832707968/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1787764270.141021:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c8400.
00000100:00000001:1.0:1787764270.141027:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764270.141028:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764270.141032:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764270.141035:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930780 x1874599832708352/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764270.141036:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764270.141038:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764270.141043:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764270.141045:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764270.141049:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00080000:3.0:1787764270.141050:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:1.0:1787764270.141053:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764270.141056:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000040:1.0:1787764270.141061:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3977950] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930780 x1874599832708352/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141067:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:1.0:1787764270.141068:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764270.141076:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21994)
00000100:00000040:3.0:1787764270.141080:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932940 x1874599832707968/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764270.141091:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832707968:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764270.141096:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764270.141099:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764270.141104:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932940 x1874599832707968/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141112:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764270.141114:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764270.141117:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764270.141121:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764270.141124:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764270.141125:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764270.141127:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764270.141129:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca400.
02000000:00000001:3.0:1787764270.141148:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764270.141150:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764270.141152:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932940.
00000100:00000001:3.0:1787764270.141155:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764270.141156:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764270.141158:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764270.141164:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764270.141165:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764270.141167:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764270.141168:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764270.141170:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764270.141174:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933840 x1874599832708096/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141179:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764270.141180:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764270.141182:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764270.141183:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764270.141185:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764270.141188:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832708096:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764270.141191:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764270.141192:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764270.141193:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764270.141195:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764270.141198:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764270.141199:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764270.141202:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764270.141204:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764270.141207:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764270.141210:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832708096, portal 10
00000100:00000001:3.0:1787764270.141211:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764270.141212:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497735744 : -115097211815872 : ffff9751d7933840)
00000100:00000040:3.0:1787764270.141217:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933840 x1874599832708096/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764295 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141222:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764270.141224:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764270.141227:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764270.141229:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832708096, offset 0
00000400:00000200:3.0:1787764270.141233:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764270.141236:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764270.141239:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764270.141242:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764270.141244:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764270.141246:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764270.141249:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764270.141252:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58767:58767:256:4294967295] 192.168.203.110@tcp LPNI seq info [58767:58767:8:4294967295]
00000400:00000200:3.0:1787764270.141255:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764270.141262:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764270.141266:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764270.141270:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764270.141275:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764270.141277:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787764270.141282:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764270.141284:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787764270.141287:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764270.141288:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764270.141291:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764270.141294:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764270.141296:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764270.141298:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764270.141300:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764270.141304:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933840 x1874599832708096/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764295 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141309:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764270.141312:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933840 x1874599832708096/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764295 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141391:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764270.141392:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764270.141396:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764270.141400:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764270.141403:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764270.141405:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316657184
00000100:00000001:3.0:1787764270.141408:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316658419
00000100:00000001:3.0:1787764270.141409:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316657184
00000100:00000001:3.0:1787764270.141411:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316658419
00000100:00000001:3.0:1787764270.141413:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764270.141415:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764270.141417:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764270.141421:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764270/real 1787764270]  req@ffff9751d7933840 x1874599832708096/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764270.141427:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764270.141428:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764270.141430:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764270.141432:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764270.141433:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764270.141434:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764270.141436:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933840 x1874599832708096/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764270.141442:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933840 x1874599832708096/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141446:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764270.141447:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764270.141448:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764270.141449:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764270.141451:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933840 x1874599832708096/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141455:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764270.141458:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933840 x1874599832708096/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141462:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764270.141464:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764270.141465:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764270.141466:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764270.141467:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764270.141476:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764270.141478:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764270.141481:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764270.141482:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764270.141485:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933840 x1874599832708096/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764270.141489:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832708096:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764270.141492:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764270.141494:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932580 x1874599832708224/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141498:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764270.141499:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764270.141500:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764270.141501:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764270.141502:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764270.141504:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832708224:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764270.141506:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764270.141508:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764270.141509:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764270.141510:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764270.141513:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764270.141514:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764270.141516:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764270.141518:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764270.141520:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764270.141522:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832708224, portal 10
00000100:00000001:3.0:1787764270.141523:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764270.141524:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497730944 : -115097211820672 : ffff9751d7932580)
00000100:00000040:3.0:1787764270.141528:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932580 x1874599832708224/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764295 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141537:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764270.141539:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764270.141540:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764270.141542:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832708224, offset 0
00000400:00000200:3.0:1787764270.141544:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764270.141547:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764270.141549:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764270.141551:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764270.141553:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764270.141554:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764270.141556:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764270.141558:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58768:58768:256:4294967295] 192.168.203.110@tcp LPNI seq info [58768:58768:8:4294967295]
00000400:00000200:3.0:1787764270.141561:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764270.141566:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764270.141569:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764270.141571:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764270.141574:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764270.141577:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787764270.141579:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764270.141581:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787764270.141584:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764270.141585:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764270.141588:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764270.141590:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764270.141592:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764270.141593:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764270.141595:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764270.141598:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932580 x1874599832708224/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764295 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141603:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764270.141605:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932580 x1874599832708224/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764295 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141614:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764270.141615:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764270.141617:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764270.141619:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764270.141621:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764270.141622:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316657172
00000100:00000001:3.0:1787764270.141624:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316658419
00000100:00000001:3.0:1787764270.141625:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316657172
00000100:00000001:3.0:1787764270.141627:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316658419
00000100:00000001:3.0:1787764270.141628:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764270.141629:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764270.141630:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764270.141632:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764270/real 1787764270]  req@ffff9751d7932580 x1874599832708224/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764270.141636:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764270.141637:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764270.141638:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764270.141640:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764270.141641:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764270.141642:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764270.141644:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932580 x1874599832708224/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764270.141649:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932580 x1874599832708224/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141653:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764270.141653:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764270.141654:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764270.141655:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764270.141657:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932580 x1874599832708224/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141662:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764270.141665:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932580 x1874599832708224/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141669:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764270.141670:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764270.141671:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764270.141672:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764270.141673:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764270.141674:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764270.141676:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764270.141677:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764270.141679:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764270.141681:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932580 x1874599832708224/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764270.141685:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832708224:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764270.141688:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764270.141689:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930780 x1874599832708352/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141693:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764270.141694:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764270.141695:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764270.141696:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764270.141698:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764270.141699:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832708352:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764270.141702:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764270.141703:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764270.141704:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764270.141705:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764270.141708:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764270.141709:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764270.141711:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764270.141713:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764270.141715:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764270.141716:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832708352, portal 4
00000100:00000001:3.0:1787764270.141718:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764270.141719:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497723264 : -115097211828352 : ffff9751d7930780)
00000100:00000040:3.0:1787764270.141722:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930780 x1874599832708352/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764295 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141727:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764270.141728:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764270.141729:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764270.141731:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832708352, offset 0
00000400:00000200:3.0:1787764270.141733:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764270.141735:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764270.141737:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764270.141738:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764270.141740:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764270.141741:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764270.141742:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764270.141744:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58769:58769:256:4294967295] 192.168.203.110@tcp LPNI seq info [58769:58769:8:4294967295]
00000400:00000200:3.0:1787764270.141747:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764270.141751:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764270.141754:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764270.141756:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764270.141758:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764270.141759:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787764270.141762:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764270.141764:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787764270.141766:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764270.141768:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764270.141770:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764270.141772:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764270.141774:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764270.141775:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764270.141776:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764270.141778:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930780 x1874599832708352/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764295 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141783:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764270.141784:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930780 x1874599832708352/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764295 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141789:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764270.141790:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764270.141791:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764270.141793:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764270.141795:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764270.141796:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316657189
00000100:00000001:3.0:1787764270.141797:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316658419
00000100:00000001:3.0:1787764270.141799:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316657189
00000100:00000001:3.0:1787764270.141800:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316658419
00000100:00000001:3.0:1787764270.141802:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764270.141803:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764270.141804:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764270.141806:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764270/real 1787764270]  req@ffff9751d7930780 x1874599832708352/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764270.141810:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764270.141811:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764270.141812:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764270.141813:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764270.141814:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764270.141816:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764270.141817:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930780 x1874599832708352/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764270.141823:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930780 x1874599832708352/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141827:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764270.141828:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764270.141829:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764270.141830:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764270.141832:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930780 x1874599832708352/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141835:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764270.141838:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930780 x1874599832708352/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141841:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764270.141843:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764270.141844:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764270.141845:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764270.141846:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764270.141846:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764270.141848:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764270.141850:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764270.141851:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764270.141853:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930780 x1874599832708352/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764270.141858:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832708352:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764270.141860:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764270.141861:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764270.141864:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933840 x1874599832708096/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141869:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764270.141870:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764270.141871:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764270.141873:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764270.141874:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764270.141875:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764270.141876:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764270.141877:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8c00.
02000000:00000001:3.0:1787764270.141879:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764270.141880:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764270.141881:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933840.
00000100:00000001:3.0:1787764270.141884:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764270.141885:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764270.141886:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764270.141888:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932580 x1874599832708224/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141892:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764270.141893:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764270.141894:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764270.141896:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764270.141896:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764270.141897:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764270.141898:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764270.141899:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8800.
02000000:00000001:3.0:1787764270.141900:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764270.141901:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764270.141901:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932580.
00000100:00000001:3.0:1787764270.141903:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764270.141903:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764270.141904:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764270.141906:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930780 x1874599832708352/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764295 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764270.141910:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764270.141911:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764270.141912:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764270.141914:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764270.141914:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764270.141915:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764270.141916:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764270.141917:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c8400.
02000000:00000001:3.0:1787764270.141918:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764270.141919:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764270.141920:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930780.
00000100:00000001:3.0:1787764270.141921:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764270.141922:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764270.141923:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764270.141925:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764270.141926:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764270.141927:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764270.141928:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764270.780246:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764270.780247:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764270.780253:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764270.780256:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764270.780259:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764270.780261:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764270.780262:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764270.780270:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764270.780270:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764270.780272:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764270.780278:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764270.780283:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764270.780286:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764270.780288:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764270.780291:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764270.780322:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764270.780327:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764270.780331:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764271.164228:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764271.164235:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764271.164237:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764271.164242:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764271.164246:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764271.164248:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764271.164250:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764271.164252:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764271.164254:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764271.420193:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316658419, at 4316658432
00000100:00000010:3.0:1787764271.420207:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40ba0.
00000100:00000040:2.0:1787764271.420245:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316658419, at 4316658432
00000100:00000010:2.0:1787764271.420255:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40ae0.
00000100:00000001:2.0:1787764271.804249:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764271.804249:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764271.804258:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764271.804260:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764271.804264:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764271.804265:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764271.804270:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764271.804273:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764271.804274:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764271.804275:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764271.804282:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764271.804288:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764271.804293:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764271.804299:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764271.804303:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764271.804311:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764271.804318:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764271.804323:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764272.188242:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764272.188252:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764272.188256:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764272.188264:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764272.188270:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764272.188274:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764272.188277:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764272.188280:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764272.188284:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764272.828239:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764272.828246:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764272.828248:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764272.828249:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764272.828252:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764272.828254:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764272.828259:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764272.828261:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764272.828264:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764272.828267:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764272.828268:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764272.828273:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764272.828274:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764272.828276:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764272.828278:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764272.828280:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764272.828284:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764272.828285:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764272.828286:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764272.828291:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764272.828293:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764272.828355:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764272.828358:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764272.828361:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764273.212242:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764273.212249:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764273.212251:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764273.212255:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764273.212260:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764273.212264:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764273.212267:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764273.212270:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764273.212273:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764273.852281:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764273.852291:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764273.852291:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764273.852293:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764273.852298:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764273.852300:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764273.852305:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764273.852306:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764273.852310:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764273.852310:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764273.852315:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764273.852319:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764273.852319:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764273.852324:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764273.852324:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764273.852386:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764273.852392:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764273.852397:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764274.236209:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764274.236215:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764274.236217:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764274.236221:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764274.236224:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764274.236228:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764274.236230:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764274.236232:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764274.236235:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764274.876243:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764274.876247:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764274.876249:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764274.876252:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764274.876252:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764274.876259:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764274.876260:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764274.876263:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764274.876265:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764274.876268:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764274.876269:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764274.876271:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764274.876272:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764274.876275:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764274.876277:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764274.876308:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764274.876312:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764274.876315:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764275.260236:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764275.260242:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764275.260244:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764275.260249:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764275.260253:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764275.260257:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1787764275.260258:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764275.260259:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764275.260264:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1787764275.260266:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764275.260267:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764275.260269:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764275.260273:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764275.260276:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764275.260282:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21989
00000400:00000200:2.0:1787764275.260287:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764275.260293:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764275.260296:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764275.260299:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764275.260301:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764275.260304:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764275.260306:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764275.260310:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764275.260311:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787764275.260313:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764275.260316:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787764275.260317:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787764275.260319:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787764275.260321:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764275.260322:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764275.260336:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764275.260338:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764275.260340:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787764275.260343:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764275.260344:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764275.260347:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764275.260351:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764275.260354:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764275.260356:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764275.260358:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764275.260368:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658780.
00000020:00000040:2.0:1787764275.260372:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787764275.260375:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764275.260377:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764275.260380:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764275.260383:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764275.260387:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764275.260395:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff975202074c00.
00000100:00000001:2.0:1787764275.260401:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764275.260408:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658780 x1874599832708480/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764275.260416:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764275.260419:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764275.260423:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000744875b5] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658780 x1874599832708480/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764275.260439:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764275.260442:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764275.260444:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787764275.260445:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764275.260447:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764275.260448:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764275.260450:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21989
00000400:00000200:2.0:1787764275.260453:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764275.260456:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764275.260458:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764275.260461:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764275.260463:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764275.260465:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764275.260466:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764275.260468:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764275.260469:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:2.0:1787764275.260470:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787764275.260471:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1787764275.260472:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:2.0:1787764275.260476:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:2.0:1787764275.260478:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:3.0:1787764275.260478:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1787764275.260483:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764275.260484:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1787764275.260485:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764275.260489:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764275.260491:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764275.260494:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00000040:3.0:1787764275.260495:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832708480/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787764275.260498:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764275.260503:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764275.260507:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764275.260512:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1787764275.260513:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764275.260514:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764275.260520:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764275.260522:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764275.260526:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764275.260526:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764275.260533:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:3.0:1787764275.260534:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832708480:192.168.203.110@tcp:250:kworker.0
00000008:00100000:2.0:1787764275.260538:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787764275.260542:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764275.260546:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764275.260549:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000010:2.0:1787764275.260552:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
02000000:00000001:3.0:1787764275.260555:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000020:00000040:2.0:1787764275.260556:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764275.260561:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764275.260561:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764275.260563:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764275.260566:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764275.260567:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:2.0:1787764275.260568:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764275.260571:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764275.260576:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff975202077800.
02000000:00000010:3.0:1787764275.260577:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
00000100:00000001:2.0:1787764275.260579:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764275.260581:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764275.260586:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832708608/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787764275.260590:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764275.260597:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000100:00000001:2.0:1787764275.260598:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764275.260601:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764275.260604:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000040:2.0:1787764275.260605:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832708608/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764275.260613:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832708480, portal 25
00000100:00000001:2.0:1787764275.260614:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764275.260618:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00080000:2.0:1787764275.260619:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764275.260621:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00080000:2.0:1787764275.260623:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1787764275.260625:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764275.260627:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000040:3.0:1787764275.260630:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832708480/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764300 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764275.260631:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764275.260637:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21989
00000400:00000200:2.0:1787764275.260645:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764275.260649:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:2.0:1787764275.260651:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764275.260657:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764275.260657:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764275.260662:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000040:2.0:1787764275.260664:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000200:3.0:1787764275.260666:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832708480, offset 0
00000100:00000001:2.0:1787764275.260667:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764275.260671:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764275.260673:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764275.260676:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:3.0:1787764275.260677:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000020:00000040:2.0:1787764275.260679:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787764275.260681:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764275.260683:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:2.0:1787764275.260684:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:2.0:1787764275.260687:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:3.0:1787764275.260688:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:2.0:1787764275.260693:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764275.260695:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787764275.260698:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000040:2.0:1787764275.260699:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764275.260703:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:3.0:1787764275.260706:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:2.0:1787764275.260707:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:2.0:1787764275.260711:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:3.0:1787764275.260714:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00000001:2.0:1787764275.260715:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764275.260718:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1787764275.260719:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
02000000:00000001:2.0:1787764275.260721:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1787764275.260724:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
02000000:00000001:2.0:1787764275.260726:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764275.260731:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1787764275.260731:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58770:58770:256:4294967295] 192.168.203.110@tcp LPNI seq info [58770:58770:8:4294967295]
00000008:00100000:2.0:1787764275.260734:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787764275.260738:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764275.260739:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764275.260743:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764275.260747:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000400:00000200:3.0:1787764275.260752:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000020:00000040:2.0:1787764275.260753:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764275.260759:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764275.260762:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787764275.260763:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787764275.260766:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764275.260768:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764275.260770:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1787764275.260770:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
02000000:00000010:2.0:1787764275.260778:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff975202077400.
00000800:00000200:3.0:1787764275.260782:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787764275.260783:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787764275.260789:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000100:00100000:2.0:1787764275.260791:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832708736/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1787764275.260800:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787764275.260805:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:3.1:1787764275.260806:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:2.0:1787764275.260809:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1787764275.260813:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000040:2.0:1787764275.260816:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832708736/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000010:3.0:1787764275.260817:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764275.260825:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1787764275.260826:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764275.260830:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1787764275.260834:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:2.0:1787764275.260836:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:2.0:1787764275.260839:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1787764275.260839:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764275.260843:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00080000:2.0:1787764275.260844:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787764275.260848:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764275.260849:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764275.260850:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:2.0:1787764275.260852:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764275.260856:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21989
00000100:00000200:3.0:1787764275.260857:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832708480/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764300 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764275.260862:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764275.260869:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787764275.260873:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764275.260878:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832708480/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764300 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787764275.260879:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764275.260886:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764275.260892:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764275.260892:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764275.260894:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764275.260896:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764275.260897:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764275.260900:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764275.260901:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000100:00000001:3.0:1787764275.260902:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764275.260903:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764275.260907:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000400:00000200:3.0:1787764275.260911:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:2.0:1787764275.260912:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:3.0:1787764275.260915:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316663539
00000020:00000001:2.0:1787764275.260916:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:2.0:1787764275.260922:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764275.260924:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000010:3.0:1787764275.260924:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40ba0.
00000100:00000040:2.0:1787764275.260926:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764275.260929:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764275.260931:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:3.0:1787764275.260933:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00080000:2.0:1787764275.260934:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764275.260938:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764275.260939:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764275.260941:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316663539
02000000:00000001:2.0:1787764275.260942:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764275.260948:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764275.260948:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40ea0.
00000100:00000001:2.0:1787764275.260950:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764275.260954:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000008:00100000:2.0:1787764275.260955:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:2.0:1787764275.260960:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764275.260960:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764275.260961:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764275.260966:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764275.260968:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000100:00000001:3.0:1787764275.260969:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000020:00000040:2.0:1787764275.260972:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:2.0:1787764275.260976:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00100000:3.0:1787764275.260977:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764275/real 1787764275]  req@ffff9751f8658780 x1874599832708480/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764275.260978:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764275.260982:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764275.260988:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000001:3.0:1787764275.260989:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:2.0:1787764275.260992:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1787764275.260993:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764275.260996:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
02000000:00000010:2.0:1787764275.260999:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff975202074000.
00000400:00000200:3.0:1787764275.261000:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764275.261004:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:2.0:1787764275.261005:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764275.261007:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00100000:2.0:1787764275.261012:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832708864/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1787764275.261014:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832708480/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764275.261033:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764275.261037:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764275.261038:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832708480/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787764275.261044:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832708864/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261048:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764275.261051:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764275.261056:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:2.0:1787764275.261058:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764275.261059:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764275.261064:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (21999)
00000100:00100000:3.0:1787764275.261067:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832708480/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261072:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764275.261076:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832708480/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261081:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764275.261083:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764275.261084:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764275.261085:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764275.261087:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764275.261088:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764275.261090:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764275.261092:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764275.261094:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764275.261097:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832708480/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764275.261102:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832708480:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764275.261107:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764275.261109:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764275.261111:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832708480/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261115:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764275.261117:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764275.261119:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764275.261121:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764275.261122:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764275.261123:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764275.261124:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764275.261126:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff975202074c00.
02000000:00000001:3.0:1787764275.261129:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764275.261131:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764275.261149:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:3.0:1787764275.261152:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764275.261153:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764275.261154:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764275.261159:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764275.261160:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764275.261162:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764275.261163:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764275.261164:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764275.261167:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832708608/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261171:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764275.261172:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764275.261174:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764275.261175:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764275.261176:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764275.261178:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832708608:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764275.261180:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764275.261182:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764275.261183:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764275.261185:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764275.261187:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764275.261189:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764275.261191:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764275.261193:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764275.261195:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764275.261197:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832708608, portal 10
00000100:00000001:3.0:1787764275.261198:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764275.261199:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:3.0:1787764275.261204:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832708608/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764300 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261208:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764275.261210:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764275.261211:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764275.261213:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832708608, offset 0
00000400:00000200:3.0:1787764275.261215:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764275.261218:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764275.261220:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764275.261222:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764275.261224:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764275.261226:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764275.261227:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764275.261229:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58771:58771:256:4294967295] 192.168.203.110@tcp LPNI seq info [58771:58771:8:4294967295]
00000400:00000200:3.0:1787764275.261232:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764275.261236:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764275.261239:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764275.261241:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764275.261244:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764275.261246:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787764275.261249:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764275.261251:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1787764275.261253:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764275.261255:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764275.261257:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764275.261259:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764275.261261:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764275.261263:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764275.261264:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764275.261266:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832708608/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764300 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261272:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764275.261274:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832708608/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764300 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261278:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764275.261279:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764275.261280:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764275.261283:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764275.261285:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764275.261286:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316662304
00000100:00000001:3.0:1787764275.261288:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316663539
00000100:00000001:3.0:1787764275.261290:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316662304
00000100:00000001:3.0:1787764275.261291:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316663539
00000100:00000001:3.0:1787764275.261293:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764275.261294:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764275.261295:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764275.261297:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764275/real 1787764275]  req@ffff9751f8658000 x1874599832708608/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764275.261302:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764275.261303:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764275.261304:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764275.261305:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764275.261306:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764275.261308:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764275.261310:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832708608/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764275.261316:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832708608/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261320:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764275.261321:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764275.261322:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764275.261323:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764275.261339:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832708608/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261344:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764275.261346:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832708608/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261351:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764275.261352:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764275.261354:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764275.261355:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764275.261356:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764275.261366:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764275.261367:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764275.261369:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764275.261371:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764275.261373:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832708608/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764275.261378:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832708608:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764275.261381:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764275.261382:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832708736/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261391:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764275.261392:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764275.261393:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764275.261395:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764275.261396:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764275.261398:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832708736:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764275.261400:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764275.261401:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764275.261402:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764275.261403:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764275.261406:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764275.261407:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764275.261409:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764275.261411:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764275.261413:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764275.261415:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832708736, portal 10
00000100:00000001:3.0:1787764275.261416:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764275.261417:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:3.0:1787764275.261421:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832708736/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764300 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261427:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764275.261429:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764275.261432:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764275.261434:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832708736, offset 0
00000400:00000200:3.0:1787764275.261438:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764275.261442:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764275.261445:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764275.261448:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764275.261451:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764275.261454:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764275.261456:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764275.261459:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58772:58772:256:4294967295] 192.168.203.110@tcp LPNI seq info [58772:58772:8:4294967295]
00000400:00000200:3.0:1787764275.261464:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764275.261470:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764275.261475:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764275.261479:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764275.261483:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764275.261485:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787764275.261488:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764275.261491:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1787764275.261493:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764275.261495:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764275.261497:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764275.261499:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764275.261502:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764275.261503:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764275.261505:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764275.261508:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832708736/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764300 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261513:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764275.261515:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832708736/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764300 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261520:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764275.261520:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764275.261523:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764275.261525:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764275.261527:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764275.261528:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316662292
00000100:00000001:3.0:1787764275.261530:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316663539
00000100:00000001:3.0:1787764275.261532:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316662292
00000100:00000001:3.0:1787764275.261533:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316663539
00000100:00000001:3.0:1787764275.261535:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764275.261538:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764275.261540:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764275.261544:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764275/real 1787764275]  req@ffff9751f8659a40 x1874599832708736/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764275.261552:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764275.261554:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764275.261556:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764275.261558:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764275.261560:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764275.261562:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764275.261566:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832708736/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764275.261574:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832708736/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261578:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764275.261579:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764275.261581:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764275.261581:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764275.261584:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832708736/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261588:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764275.261591:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832708736/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261595:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764275.261597:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764275.261598:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764275.261599:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764275.261600:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764275.261601:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764275.261603:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764275.261606:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764275.261607:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764275.261609:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832708736/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764275.261614:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832708736:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764275.261617:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764275.261619:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832708864/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261623:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764275.261623:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764275.261625:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764275.261626:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764275.261627:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764275.261629:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832708864:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764275.261631:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764275.261632:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764275.261634:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764275.261635:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764275.261637:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764275.261639:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764275.261642:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764275.261643:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764275.261646:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764275.261648:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832708864, portal 4
00000100:00000001:3.0:1787764275.261649:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764275.261650:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:3.0:1787764275.261654:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832708864/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764300 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261658:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764275.261661:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764275.261662:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764275.261663:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832708864, offset 0
00000400:00000200:3.0:1787764275.261666:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764275.261669:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764275.261671:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764275.261673:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764275.261675:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764275.261677:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764275.261678:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764275.261680:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58773:58773:256:4294967295] 192.168.203.110@tcp LPNI seq info [58773:58773:8:4294967295]
00000400:00000200:3.0:1787764275.261683:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764275.261687:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764275.261690:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764275.261692:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764275.261694:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764275.261696:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787764275.261699:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764275.261701:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1787764275.261703:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764275.261704:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764275.261706:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764275.261709:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764275.261710:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764275.261712:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764275.261713:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764275.261715:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832708864/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764300 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261720:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764275.261722:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832708864/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764300 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261726:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764275.261727:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764275.261730:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764275.261732:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764275.261734:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764275.261735:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316662309
00000100:00000001:3.0:1787764275.261736:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316663539
00000100:00000001:3.0:1787764275.261738:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316662309
00000100:00000001:3.0:1787764275.261740:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316663539
00000100:00000001:3.0:1787764275.261741:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764275.261742:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764275.261744:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764275.261746:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764275/real 1787764275]  req@ffff9751f86583c0 x1874599832708864/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764275.261750:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764275.261751:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764275.261752:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764275.261753:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764275.261754:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764275.261755:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764275.261757:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832708864/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764275.261762:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832708864/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261766:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764275.261767:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764275.261768:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764275.261769:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764275.261771:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832708864/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261775:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764275.261777:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832708864/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261782:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764275.261784:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764275.261785:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764275.261786:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764275.261787:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764275.261788:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764275.261789:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764275.261791:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764275.261792:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764275.261795:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832708864/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764275.261799:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832708864:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764275.261801:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764275.261802:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764275.261804:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832708608/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261808:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764275.261809:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764275.261810:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764275.261812:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764275.261812:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764275.261813:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764275.261815:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764275.261816:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff975202077800.
02000000:00000001:3.0:1787764275.261817:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764275.261818:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764275.261819:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:3.0:1787764275.261821:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764275.261821:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764275.261823:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764275.261825:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832708736/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261829:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764275.261830:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764275.261831:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764275.261832:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764275.261833:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764275.261833:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764275.261834:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764275.261835:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff975202077400.
02000000:00000001:3.0:1787764275.261836:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764275.261837:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764275.261838:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:3.0:1787764275.261839:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764275.261840:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764275.261841:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764275.261842:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832708864/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764300 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764275.261846:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764275.261847:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764275.261848:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764275.261849:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764275.261850:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764275.261850:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764275.261851:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764275.261852:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff975202074000.
02000000:00000001:3.0:1787764275.261853:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764275.261854:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764275.261855:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:3.0:1787764275.261856:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764275.261856:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764275.261858:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764275.261860:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764275.261861:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764275.261862:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764275.261863:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764275.900220:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764275.900222:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764275.900228:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764275.900229:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764275.900231:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764275.900232:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00100000:2.0:1787764275.900238:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764275.900241:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764275.900244:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764275.900245:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764275.900251:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764275.900252:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764275.900253:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764275.900255:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764275.900257:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764275.900258:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764275.900262:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764275.900266:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764275.900267:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764275.900270:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764275.900271:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764275.900314:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764275.900317:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764275.900336:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764276.284245:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764276.284253:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764276.284255:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764276.284261:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764276.284266:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764276.284269:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764276.284272:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764276.284275:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764276.284278:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764276.540210:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316663539, at 4316663552
00000100:00000010:3.0:1787764276.540224:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40ea0.
00000100:00000040:2.0:1787764276.540253:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316663539, at 4316663552
00000100:00000010:2.0:1787764276.540261:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40ba0.
00010000:00000001:3.0:1787764276.924219:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764276.924224:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764276.924226:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764276.924229:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764276.924232:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764276.924236:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764276.924236:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764276.924240:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764276.924241:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764276.924248:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764276.924253:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764276.924256:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764276.924259:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764276.924262:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764276.924266:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764276.924279:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764276.924283:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764276.924287:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764277.308249:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764277.308255:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764277.308257:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764277.308262:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764277.308268:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764277.308273:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764277.308277:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764277.308283:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764277.308288:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764277.948228:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764277.948232:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764277.948234:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764277.948237:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764277.948238:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764277.948246:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764277.948246:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764277.948251:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764277.948252:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764277.948255:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764277.948255:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764277.948260:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764277.948261:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764277.948263:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764277.948266:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764277.948267:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764277.948268:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764277.948272:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764277.948274:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764277.948277:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764277.948278:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764277.948336:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764277.948340:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764277.948361:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764278.332238:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764278.332244:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764278.332246:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764278.332251:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764278.332254:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764278.332257:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764278.332259:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764278.332262:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764278.332264:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764278.972255:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764278.972260:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764278.972262:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764278.972265:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764278.972265:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764278.972273:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764278.972273:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764278.972276:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764278.972279:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764278.972282:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764278.972284:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764278.972285:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764278.972289:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764278.972293:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764278.972295:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764278.972345:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764278.972349:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764278.972351:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764279.356235:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764279.356242:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764279.356243:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764279.356248:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764279.356251:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764279.356253:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764279.356255:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764279.356258:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764279.356261:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764279.996251:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764279.996258:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764279.996263:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764279.996264:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764279.996267:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764279.996268:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764279.996323:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764279.996326:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764279.996329:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764279.996375:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764279.996380:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764279.996382:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764279.996385:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764279.996387:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764279.996390:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764279.996392:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764279.996394:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764279.996396:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764280.380241:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764280.380250:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764280.380253:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764280.380259:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:0.0:1787764280.380263:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764280.380264:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764280.380269:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787764280.380270:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787764280.380274:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1787764280.380275:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:0.0:1787764280.380278:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764280.380280:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764280.380283:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764280.380285:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764280.380302:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 21994
00000400:00000200:0.0:1787764280.380310:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764280.380316:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764280.380318:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764280.380321:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764280.380323:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764280.380325:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764280.380327:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764280.380330:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764280.380331:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764280.380334:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764280.380337:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764280.380338:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764280.380340:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787764280.380341:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764280.380343:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764280.380344:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764280.380346:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764280.380348:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787764280.380350:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764280.380353:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764280.380356:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764280.380360:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764280.380363:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764280.380364:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764280.380367:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764280.380378:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787764280.380382:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764280.380386:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764280.380387:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764280.380390:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764280.380393:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764280.380397:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764280.380406:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787764280.380413:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764280.380421:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832708992/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764280.380432:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764280.380436:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764280.380440:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832708992/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764280.380460:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764280.380463:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764280.380466:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764280.380468:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764280.380470:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764280.380473:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764280.380477:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21994
00000100:00000001:3.0:1787764280.380480:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000400:00000200:0.0:1787764280.380483:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764280.380485:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764280.380488:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000400:00000200:0.0:1787764280.380491:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764280.380495:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:3.0:1787764280.380496:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e940 x1874599832708992/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764280.380503:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764280.380509:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764280.380510:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764280.380511:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764280.380517:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:3.0:1787764280.380517:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764280.380522:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764280.380523:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:3.0:1787764280.380528:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764280.380529:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764280.380533:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787764280.380537:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00100000:3.0:1787764280.380537:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832708992:192.168.203.110@tcp:250:kworker.0
00000020:00000040:0.0:1787764280.380540:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787764280.380545:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:3.0:1787764280.380546:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000020:00000001:0.0:1787764280.380548:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:3.0:1787764280.380553:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:0.0:1787764280.380554:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:3.0:1787764280.380557:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764280.380559:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764280.380563:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:3.0:1787764280.380563:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1787764280.380568:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000010:3.0:1787764280.380571:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
00000020:00000040:0.0:1787764280.380572:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:3.0:1787764280.380577:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764280.380580:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:3.0:1787764280.380585:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764280.380586:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764280.380589:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
02000000:00000001:0.0:1787764280.380590:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764280.380597:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:3.0:1787764280.380598:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
02000000:00000001:0.0:1787764280.380604:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764280.380606:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832708992, portal 25
00000100:00000001:0.0:1787764280.380608:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764280.380612:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764280.380614:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049528640 : -115096660022976 : ffff9751f876e940)
00000008:00100000:0.0:1787764280.380615:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787764280.380621:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764280.380621:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e940 x1874599832708992/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764305 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764280.380628:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764280.380633:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000020:00000040:0.0:1787764280.380638:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000100:00000001:3.0:1787764280.380642:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:0.0:1787764280.380644:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:3.0:1787764280.380647:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
02000000:00000001:0.0:1787764280.380648:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764280.380651:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764280.380654:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832708992, offset 0
00000100:00000001:0.0:1787764280.380662:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1787764280.380662:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
02000000:00000001:0.0:1787764280.380665:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764280.380667:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764280.380670:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000010:0.0:1787764280.380675:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000400:00000200:3.0:1787764280.380678:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:0.0:1787764280.380681:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764280.380684:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:0.0:1787764280.380690:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832709120/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787764280.380690:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764280.380696:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764280.380701:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:0.0:1787764280.380706:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1787764280.380708:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58774:58774:256:4294967295] 192.168.203.110@tcp LPNI seq info [58774:58774:8:4294967295]
00000100:00000001:0.0:1787764280.380709:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764280.380718:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832709120/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764280.380718:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764280.380729:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764280.380734:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1787764280.380734:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00080000:0.0:1787764280.380741:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787764280.380745:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764280.380746:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000200:3.0:1787764280.380751:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787764280.380752:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764280.380757:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21994
00000400:00000200:0.0:1787764280.380760:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000010:3.0:1787764280.380762:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000400:00000200:0.0:1787764280.380764:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764280.380767:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764280.380769:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764280.380772:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1787764280.380775:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787764280.380776:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764280.380780:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:3.0:1787764280.380781:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000020:00000001:0.0:1787764280.380782:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764280.380784:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787764280.380786:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764280.380789:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787764280.380791:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787764280.380795:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:3.1:1787764280.380796:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787764280.380799:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764280.380803:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764280.380804:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000200:3.1:1787764280.380804:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000100:00000001:0.0:1787764280.380807:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764280.380812:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787764280.380817:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000800:00000100:3.0:1787764280.380817:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787764280.380822:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764280.380824:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000010:3.0:1787764280.380825:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
02000000:00000001:0.0:1787764280.380827:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764280.380830:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764280.380831:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1787764280.380832:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764280.380835:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787764280.380837:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764280.380839:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764280.380841:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
00000020:00000040:0.0:1787764280.380843:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000400:00000200:3.0:1787764280.380847:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:0.0:1787764280.380848:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764280.380852:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764280.380854:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1787764280.380854:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
02000000:00000001:0.0:1787764280.380856:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:3.0:1787764280.380858:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
02000000:00000001:0.0:1787764280.380859:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764280.380864:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:3.0:1787764280.380865:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1787764280.380867:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764280.380871:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e940 x1874599832708992/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764305 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787764280.380874:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832709248/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764280.380896:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764280.380899:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764280.380906:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832709248/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.380908:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764280.380915:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e940 x1874599832708992/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764305 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764280.380916:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764280.380921:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787764280.380927:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1787764280.380927:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764280.380931:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:0.0:1787764280.380932:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764280.380936:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787764280.380937:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1787764280.380938:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764280.380939:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764280.380943:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1787764280.380945:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:0.0:1787764280.380947:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 21994
00000100:00000001:3.0:1787764280.380950:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316668659
00000400:00000200:0.0:1787764280.380952:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764280.380957:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764280.380958:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764280.380961:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000010:3.0:1787764280.380961:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40ea0.
00000100:00000001:0.0:1787764280.380967:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764280.380968:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764280.380969:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764280.380971:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1787764280.380972:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000020:00000040:0.0:1787764280.380973:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787764280.380976:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764280.380978:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787764280.380980:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787764280.380982:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:3.0:1787764280.380983:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316668659
00000100:00000040:0.0:1787764280.380987:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764280.380989:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000010:3.0:1787764280.380990:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40660.
00000100:00000040:0.0:1787764280.380991:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764280.380996:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764280.380998:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000001:3.0:1787764280.380998:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00080000:0.0:1787764280.381004:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764280.381004:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764280.381007:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764280.381008:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764280.381009:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764280.381013:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000001:0.0:1787764280.381014:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764280.381016:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764280.381018:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:3.0:1787764280.381020:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764280/real 1787764280]  req@ffff9751f876e940 x1874599832708992/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:0.0:1787764280.381021:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787764280.381026:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764280.381030:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764280.381036:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000020:00000040:0.0:1787764280.381039:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787764280.381041:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764280.381042:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1787764280.381042:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:0.0:1787764280.381047:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000001:3.0:1787764280.381047:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:0.0:1787764280.381050:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:3.0:1787764280.381052:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
02000000:00000001:0.0:1787764280.381053:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764280.381060:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000400:00000200:3.0:1787764280.381060:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000100:00000001:0.0:1787764280.381062:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764280.381065:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00100000:0.0:1787764280.381069:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832709376/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764280.381071:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764280.381078:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e940 x1874599832708992/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764280.381079:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764280.381083:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764280.381091:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832709376/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764280.381096:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e940 x1874599832708992/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764280.381105:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764280.381111:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000040:0.0:1787764280.381112:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (22005)
00000100:00000001:3.0:1787764280.381116:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764280.381119:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764280.381120:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764280.381123:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e940 x1874599832708992/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381128:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764280.381131:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e940 x1874599832708992/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381155:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764280.381157:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764280.381158:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764280.381159:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764280.381161:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764280.381163:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764280.381165:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764280.381168:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764280.381169:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764280.381172:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e940 x1874599832708992/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764280.381178:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832708992:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764280.381183:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764280.381185:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764280.381189:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e940 x1874599832708992/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381196:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764280.381198:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764280.381201:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764280.381205:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764280.381207:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764280.381209:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764280.381211:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764280.381214:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:3.0:1787764280.381218:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764280.381220:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764280.381222:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e940.
00000100:00000001:3.0:1787764280.381226:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764280.381227:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764280.381229:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764280.381235:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764280.381237:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764280.381239:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764280.381241:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764280.381242:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764280.381246:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cf00 x1874599832709120/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381254:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764280.381255:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764280.381258:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764280.381260:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764280.381262:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764280.381266:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832709120:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764280.381269:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764280.381272:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764280.381274:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764280.381275:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764280.381278:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764280.381280:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764280.381283:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764280.381284:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764280.381287:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764280.381303:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832709120, portal 10
00000100:00000001:3.0:1787764280.381305:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764280.381306:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049521920 : -115096660029696 : ffff9751f876cf00)
00000100:00000040:3.0:1787764280.381311:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cf00 x1874599832709120/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764305 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381317:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764280.381318:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764280.381320:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764280.381322:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832709120, offset 0
00000400:00000200:3.0:1787764280.381325:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764280.381328:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764280.381331:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764280.381333:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764280.381335:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764280.381337:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764280.381339:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764280.381341:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58775:58775:256:4294967295] 192.168.203.110@tcp LPNI seq info [58775:58775:8:4294967295]
00000400:00000200:3.0:1787764280.381344:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764280.381348:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764280.381351:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764280.381353:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764280.381356:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764280.381358:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787764280.381361:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764280.381363:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:3.0:1787764280.381366:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764280.381367:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764280.381370:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764280.381372:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764280.381374:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764280.381375:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764280.381377:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764280.381379:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cf00 x1874599832709120/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764305 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381385:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764280.381387:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cf00 x1874599832709120/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764305 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381396:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764280.381397:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764280.381399:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764280.381402:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764280.381404:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764280.381406:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316667424
00000100:00000001:3.0:1787764280.381408:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316668659
00000100:00000001:3.0:1787764280.381409:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316667424
00000100:00000001:3.0:1787764280.381411:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316668659
00000100:00000001:3.0:1787764280.381412:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764280.381413:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764280.381415:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764280.381417:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764280/real 1787764280]  req@ffff9751f876cf00 x1874599832709120/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764280.381422:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764280.381423:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764280.381424:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764280.381425:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764280.381427:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764280.381428:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764280.381430:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cf00 x1874599832709120/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764280.381435:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cf00 x1874599832709120/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381439:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764280.381440:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764280.381441:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764280.381442:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764280.381444:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cf00 x1874599832709120/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381449:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764280.381452:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cf00 x1874599832709120/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381456:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764280.381458:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764280.381459:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764280.381459:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764280.381460:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764280.381470:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764280.381471:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764280.381473:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764280.381475:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764280.381477:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cf00 x1874599832709120/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764280.381482:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832709120:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764280.381484:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764280.381486:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f840 x1874599832709248/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381491:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764280.381492:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764280.381493:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764280.381494:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764280.381496:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764280.381497:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832709248:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764280.381499:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764280.381500:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764280.381502:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764280.381503:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764280.381506:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764280.381507:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764280.381509:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764280.381510:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764280.381512:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764280.381514:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832709248, portal 10
00000100:00000001:3.0:1787764280.381515:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764280.381516:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049532480 : -115096660019136 : ffff9751f876f840)
00000100:00000040:3.0:1787764280.381522:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f840 x1874599832709248/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764305 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381529:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764280.381531:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764280.381534:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764280.381536:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832709248, offset 0
00000400:00000200:3.0:1787764280.381540:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764280.381544:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764280.381548:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764280.381551:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764280.381553:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764280.381556:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764280.381559:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764280.381562:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58776:58776:256:4294967295] 192.168.203.110@tcp LPNI seq info [58776:58776:8:4294967295]
00000400:00000200:3.0:1787764280.381567:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764280.381573:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764280.381577:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764280.381579:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764280.381582:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764280.381584:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787764280.381587:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764280.381589:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:3.0:1787764280.381591:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764280.381593:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764280.381595:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764280.381598:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764280.381600:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764280.381601:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764280.381603:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764280.381606:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f840 x1874599832709248/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764305 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381611:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764280.381613:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f840 x1874599832709248/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764305 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381617:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764280.381618:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764280.381620:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764280.381623:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764280.381625:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764280.381626:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316667412
00000100:00000001:3.0:1787764280.381628:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316668659
00000100:00000001:3.0:1787764280.381629:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316667412
00000100:00000001:3.0:1787764280.381631:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316668659
00000100:00000001:3.0:1787764280.381632:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764280.381633:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764280.381634:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764280.381637:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764280/real 1787764280]  req@ffff9751f876f840 x1874599832709248/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764280.381642:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764280.381643:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764280.381644:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764280.381645:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764280.381646:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764280.381648:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764280.381650:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f840 x1874599832709248/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764280.381655:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f840 x1874599832709248/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381659:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764280.381660:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764280.381661:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764280.381662:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764280.381664:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f840 x1874599832709248/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381668:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764280.381670:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f840 x1874599832709248/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381675:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764280.381676:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764280.381677:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764280.381678:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764280.381680:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764280.381681:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764280.381682:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764280.381684:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764280.381685:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764280.381688:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f840 x1874599832709248/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764280.381692:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832709248:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764280.381695:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764280.381697:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832709376/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381702:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764280.381703:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764280.381704:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764280.381705:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764280.381707:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764280.381708:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832709376:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764280.381710:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764280.381711:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764280.381713:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764280.381714:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764280.381716:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764280.381718:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764280.381720:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764280.381721:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764280.381723:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764280.381726:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832709376, portal 4
00000100:00000001:3.0:1787764280.381728:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764280.381729:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
00000100:00000040:3.0:1787764280.381736:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832709376/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764305 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381742:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764280.381744:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764280.381747:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764280.381750:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832709376, offset 0
00000400:00000200:3.0:1787764280.381754:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764280.381757:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764280.381761:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764280.381764:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764280.381767:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764280.381770:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764280.381772:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764280.381776:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58777:58777:256:4294967295] 192.168.203.110@tcp LPNI seq info [58777:58777:8:4294967295]
00000400:00000200:3.0:1787764280.381779:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764280.381785:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764280.381790:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764280.381794:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764280.381798:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764280.381801:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000200:3.1:1787764280.381805:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764280.381808:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 6 secs later)
00000800:00000100:3.0:1787764280.381812:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764280.381814:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764280.381817:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764280.381822:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764280.381825:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764280.381827:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764280.381830:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764280.381833:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876ed00 x1874599832709376/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764305 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381839:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764280.381841:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832709376/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764305 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381845:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764280.381846:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764280.381848:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764280.381850:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764280.381853:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764280.381854:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316667429
00000100:00000001:3.0:1787764280.381856:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316668659
00000100:00000001:3.0:1787764280.381858:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316667429
00000100:00000001:3.0:1787764280.381859:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316668659
00000100:00000001:3.0:1787764280.381861:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764280.381863:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764280.381864:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764280.381866:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764280/real 1787764280]  req@ffff9751f876ed00 x1874599832709376/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764280.381872:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764280.381873:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764280.381874:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764280.381875:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764280.381876:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764280.381877:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764280.381879:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832709376/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764280.381885:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832709376/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381888:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764280.381889:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764280.381890:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764280.381891:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764280.381893:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832709376/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381897:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764280.381900:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832709376/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381903:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764280.381905:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764280.381906:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764280.381907:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764280.381909:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764280.381910:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764280.381911:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764280.381914:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764280.381915:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764280.381918:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832709376/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764280.381923:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832709376:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764280.381925:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764280.381926:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764280.381928:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cf00 x1874599832709120/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381932:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764280.381933:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764280.381934:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764280.381936:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764280.381937:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764280.381938:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764280.381940:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764280.381941:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
02000000:00000001:3.0:1787764280.381943:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764280.381944:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764280.381945:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cf00.
00000100:00000001:3.0:1787764280.381947:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764280.381948:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764280.381949:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764280.381953:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f840 x1874599832709248/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381959:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764280.381961:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764280.381963:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764280.381965:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764280.381967:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764280.381968:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764280.381970:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764280.381972:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016c00.
02000000:00000001:3.0:1787764280.381975:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764280.381976:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764280.381977:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f840.
00000100:00000001:3.0:1787764280.381980:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764280.381981:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764280.381983:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764280.381987:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832709376/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764305 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764280.381994:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764280.381996:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764280.381998:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764280.382000:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764280.382001:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764280.382002:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764280.382003:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764280.382005:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017800.
02000000:00000001:3.0:1787764280.382007:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764280.382008:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764280.382008:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:3.0:1787764280.382010:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764280.382011:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764280.382012:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764280.382015:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764280.382016:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764280.382017:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764280.382018:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764281.020260:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764281.020266:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764281.020268:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764281.020270:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764281.020273:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764281.020275:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764281.020281:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764281.020283:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764281.020298:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764281.020302:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764281.020302:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764281.020306:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764281.020310:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764281.020311:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764281.020315:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764281.020335:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764281.020339:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764281.020343:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764281.404217:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764281.404224:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764281.404226:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764281.404230:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764281.404235:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764281.404237:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764281.404239:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764281.404241:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764281.404243:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764281.660252:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316668659, at 4316668672
00000100:00000010:3.0:1787764281.660271:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40660.
00000100:00000040:2.0:1787764281.660364:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316668659, at 4316668672
00000100:00000010:2.0:1787764281.660372:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40ea0.
00000100:00000001:2.0:1787764282.044209:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764282.044213:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764282.044218:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764282.044219:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764282.044225:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764282.044225:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764282.044228:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764282.044232:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764282.044233:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764282.044235:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764282.044240:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764282.044245:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764282.044248:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764282.044251:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764282.044254:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764282.044326:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764282.044330:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764282.044333:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764282.428290:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764282.428297:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764282.428299:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764282.428303:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764282.428307:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764282.428310:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764282.428327:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764282.428330:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764282.428332:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764283.068280:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764283.068292:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764283.068292:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764283.068295:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764283.068299:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764283.068302:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764283.068308:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764283.068309:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764283.068312:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764283.068313:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764283.068319:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764283.068322:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764283.068325:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764283.068327:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764283.068333:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764283.068337:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764283.068342:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764283.068347:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764283.068350:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764283.068352:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764283.068355:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764283.068422:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764283.068427:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764283.068430:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764283.452264:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764283.452287:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764283.452289:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764283.452296:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764283.452301:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764283.452304:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764283.452307:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764283.452309:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764283.452312:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764284.092245:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764284.092249:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764284.092255:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764284.092257:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764284.092261:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764284.092277:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764284.092277:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764284.092286:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764284.092287:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764284.092289:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764284.092294:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764284.092299:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764284.092302:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764284.092304:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764284.092308:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764284.092317:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764284.092322:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764284.092327:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764284.476282:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764284.476293:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764284.476297:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764284.476303:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764284.476308:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764284.476312:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764284.476316:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764284.476320:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764284.476324:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764285.116278:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764285.116288:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764285.116288:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764285.116301:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764285.116302:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764285.116310:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764285.116311:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764285.116330:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764285.116331:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764285.116338:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764285.116340:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764285.116352:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764285.116358:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764285.116361:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764285.116366:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764285.116391:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764285.116397:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764285.116401:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764285.500230:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764285.500236:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764285.500238:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764285.500244:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764285.500247:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764285.500250:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764285.500252:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764285.500269:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787764285.500270:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764285.500272:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764285.500280:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787764285.500283:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764285.500288:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764285.500293:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764285.500300:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 22000
00000400:00000200:0.0:1787764285.500308:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764285.500315:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764285.500319:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764285.500322:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764285.500326:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764285.500329:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764285.500331:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764285.500336:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764285.500338:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764285.500341:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764285.500345:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764285.500359:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764285.500362:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787764285.500366:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764285.500369:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764285.500371:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764285.500373:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764285.500376:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787764285.500379:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764285.500382:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764285.500385:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764285.500388:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764285.500391:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764285.500393:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764285.500396:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764285.500404:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787764285.500407:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764285.500412:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764285.500414:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764285.500417:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764285.500419:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764285.500423:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764285.500431:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:0.0:1787764285.500437:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764285.500447:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832709504/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764285.500457:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764285.500462:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764285.500467:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832709504/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764285.500488:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764285.500492:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764285.500495:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764285.500497:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764285.500500:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764285.500503:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764285.500507:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22000
00000400:00000200:0.0:1787764285.500512:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764285.500516:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787764285.500519:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:0.0:1787764285.500520:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787764285.500525:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:0.0:1787764285.500526:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764285.500532:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764285.500532:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:0.0:1787764285.500536:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764285.500541:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:3.0:1787764285.500542:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f0c0 x1874599832709504/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787764285.500546:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764285.500549:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787764285.500554:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764285.500559:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000001:3.0:1787764285.500559:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764285.500561:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764285.500567:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:3.0:1787764285.500567:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764285.500571:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:0.0:1787764285.500572:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:3.0:1787764285.500574:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764285.500582:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832709504:192.168.203.110@tcp:250:kworker.0
00000100:00000040:0.0:1787764285.500584:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764285.500587:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1787764285.500589:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1787764285.500592:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:0.0:1787764285.500595:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:3.0:1787764285.500597:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764285.500601:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1787764285.500602:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764285.500606:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000010:3.0:1787764285.500607:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764285.500611:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764285.500612:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764285.500618:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764285.500619:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787764285.500623:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:3.0:1787764285.500625:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
02000000:00000001:0.0:1787764285.500630:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:3.0:1787764285.500632:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
02000000:00000001:0.0:1787764285.500636:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764285.500640:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832709504, portal 25
00000100:00000001:0.0:1787764285.500641:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764285.500645:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764285.500648:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049530560 : -115096660021056 : ffff9751f876f0c0)
00000008:00100000:0.0:1787764285.500649:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000040:3.0:1787764285.500653:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f0c0 x1874599832709504/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764310 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.500659:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764285.500662:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764285.500664:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764285.500666:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832709504, offset 0
00000008:00000001:0.0:1787764285.500668:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764285.500672:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764285.500673:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000010:0.0:1787764285.500680:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787764285.500685:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000200:3.0:1787764285.500688:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787764285.500693:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764285.500696:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764285.500697:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:0.0:1787764285.500702:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1787764285.500704:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:0.0:1787764285.500707:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1787764285.500710:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
02000000:00000001:0.0:1787764285.500712:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764285.500714:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764285.500718:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
02000000:00000010:0.0:1787764285.500721:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000400:00000200:3.0:1787764285.500721:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58778:58778:256:4294967295] 192.168.203.110@tcp LPNI seq info [58778:58778:8:4294967295]
00000400:00000200:3.0:1787764285.500728:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764285.500729:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764285.500739:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00100000:0.0:1787764285.500742:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832709632/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.0:1787764285.500746:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764285.500753:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000100:00000001:0.0:1787764285.500759:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:3.0:1787764285.500763:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787764285.500765:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787764285.500768:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000100:00000040:0.0:1787764285.500774:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832709632/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.1:1787764285.500775:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764285.500781:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:3.0:1787764285.500791:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764285.500795:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000100:00000001:0.0:1787764285.500800:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764285.500801:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:0.0:1787764285.500808:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1787764285.500809:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764285.500816:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000100:00080000:0.0:1787764285.500817:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000010:3.0:1787764285.500819:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:0.0:1787764285.500821:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1787764285.500824:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:0.0:1787764285.500825:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:3.0:1787764285.500830:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f0c0 x1874599832709504/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764310 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764285.500832:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764285.500839:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22000
00000100:00000001:3.0:1787764285.500842:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000400:00000200:0.0:1787764285.500847:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:3.0:1787764285.500851:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f0c0 x1874599832709504/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764310 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787764285.500855:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764285.500861:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764285.500868:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764285.500872:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764285.500873:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764285.500878:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764285.500878:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787764285.500883:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764285.500886:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1787764285.500887:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787764285.500889:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787764285.500892:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764285.500894:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787764285.500896:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787764285.500898:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:3.0:1787764285.500903:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:0.0:1787764285.500904:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764285.500908:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316673779
00000100:00000001:0.0:1787764285.500910:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764285.500914:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000010:3.0:1787764285.500915:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40660.
00000100:00000001:0.0:1787764285.500917:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764285.500922:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00000001:3.0:1787764285.500923:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00080000:0.0:1787764285.500927:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764285.500931:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764285.500932:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316673779
02000000:00000001:0.0:1787764285.500934:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000010:3.0:1787764285.500936:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40f60.
02000000:00000001:0.0:1787764285.500939:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764285.500940:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
02000000:00000001:0.0:1787764285.500946:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764285.500947:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764285.500952:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764285.500952:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1787764285.500957:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787764285.500957:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000008:00000001:0.0:1787764285.500962:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764285.500964:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764285/real 1787764285]  req@ffff9751f876f0c0 x1874599832709504/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764285.500965:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764285.500973:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000400:00000001:3.0:1787764285.500975:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000020:00000040:0.0:1787764285.500978:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000400:00000001:3.0:1787764285.500978:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764285.500983:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
02000000:00000001:0.0:1787764285.500984:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764285.500988:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764285.500988:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000100:00000001:0.0:1787764285.500991:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000010:3.0:1787764285.500993:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
02000000:00000001:0.0:1787764285.500995:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764285.500999:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764285.501000:0:257575:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000010:0.0:1787764285.501005:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000200:3.0:1787764285.501007:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f0c0 x1874599832709504/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764285.501011:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764285.501023:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832709760/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1787764285.501028:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f0c0 x1874599832709504/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764285.501040:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764285.501042:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787764285.501044:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764285.501045:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764285.501049:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764285.501051:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787764285.501055:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832709760/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764285.501058:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f0c0 x1874599832709504/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.501085:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764285.501090:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764285.501092:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f0c0 x1874599832709504/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764285.501095:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764285.501101:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000040:0.0:1787764285.501104:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1787764285.501105:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764285.501108:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764285.501111:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787764285.501112:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764285.501113:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764285.501116:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:0.0:1787764285.501119:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00080000:3.0:1787764285.501120:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:0.0:1787764285.501123:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:3.0:1787764285.501127:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00080000:0.0:1787764285.501128:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764285.501130:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:0.0:1787764285.501154:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764285.501158:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22000
00000400:00000200:0.0:1787764285.501166:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:3.0:1787764285.501170:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f0c0 x1874599832709504/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787764285.501173:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764285.501180:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764285.501187:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764285.501192:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:3.0:1787764285.501194:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832709504:192.168.203.110@tcp:250:kworker.0
00000100:00000040:0.0:1787764285.501197:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764285.501204:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764285.501205:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1787764285.501206:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764285.501210:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000100:00000001:3.0:1787764285.501211:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000001:0.0:1787764285.501213:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764285.501217:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000040:3.0:1787764285.501219:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f0c0 x1874599832709504/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787764285.501221:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787764285.501227:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787764285.501235:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764285.501236:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
00000100:00000001:0.0:1787764285.501241:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1787764285.501241:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764285.501246:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
00000100:00000040:0.0:1787764285.501247:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:3.0:1787764285.501250:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000100:00000001:0.0:1787764285.501252:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:3.0:1787764285.501267:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764285.501269:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000040:0.0:1787764285.501273:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000020:00000001:3.0:1787764285.501273:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764285.501277:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017000.
00000100:00080000:0.0:1787764285.501281:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:3.0:1787764285.501283:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00000001:0.0:1787764285.501286:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764285.501286:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764285.501289:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f0c0.
02000000:00000001:0.0:1787764285.501291:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764285.501293:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764285.501295:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787764285.501296:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764285.501297:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787764285.501303:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764285.501306:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:0.0:1787764285.501307:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764285.501309:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1787764285.501312:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1787764285.501312:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764285.501315:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764285.501318:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000008:00000001:0.0:1787764285.501319:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764285.501324:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764285.501324:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832709632/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:0.0:1787764285.501333:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000100:00000001:3.0:1787764285.501334:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764285.501337:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787764285.501339:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:3.0:1787764285.501341:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787764285.501345:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764285.501346:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764285.501350:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764285.501352:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764285.501355:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764285.501359:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00100000:3.0:1787764285.501359:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832709632:192.168.203.110@tcp:38:kworker.0
02000000:00000001:0.0:1787764285.501365:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764285.501367:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000010:0.0:1787764285.501372:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
02000000:00000001:3.0:1787764285.501372:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764285.501375:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764285.501378:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764285.501379:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764285.501385:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
00000100:00100000:0.0:1787764285.501389:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832709888/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1787764285.501390:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764285.501397:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764285.501402:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000100:00000001:0.0:1787764285.501404:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764285.501407:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764285.501407:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764285.501413:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832709632, portal 10
00000100:00000040:0.0:1787764285.501415:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832709888/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.501416:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764285.501419:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
00000100:00000001:0.0:1787764285.501428:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764285.501429:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832709632/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764310 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764285.501432:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (22010)
00000100:00000001:3.0:1787764285.501444:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764285.501446:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764285.501448:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764285.501450:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832709632, offset 0
00000400:00000200:3.0:1787764285.501454:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764285.501457:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764285.501460:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764285.501462:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764285.501464:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764285.501466:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764285.501468:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764285.501470:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58779:58779:256:4294967295] 192.168.203.110@tcp LPNI seq info [58779:58779:8:4294967295]
00000400:00000200:3.0:1787764285.501473:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764285.501477:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764285.501481:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764285.501484:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764285.501489:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764285.501492:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787764285.501496:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764285.501499:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:3.0:1787764285.501503:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764285.501505:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764285.501509:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764285.501512:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764285.501516:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764285.501518:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764285.501521:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764285.501525:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876c000 x1874599832709632/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764310 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.501534:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764285.501537:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832709632/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764310 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.501542:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764285.501543:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764285.501546:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764285.501549:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764285.501551:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764285.501552:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316672544
00000100:00000001:3.0:1787764285.501554:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316673779
00000100:00000001:3.0:1787764285.501556:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316672544
00000100:00000001:3.0:1787764285.501557:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316673779
00000100:00000001:3.0:1787764285.501559:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764285.501560:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764285.501561:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764285.501564:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764285/real 1787764285]  req@ffff9751f876c000 x1874599832709632/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764285.501570:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764285.501571:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764285.501572:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764285.501573:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764285.501574:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764285.501576:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764285.501578:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832709632/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764285.501583:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832709632/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.501587:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764285.501588:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764285.501589:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764285.501590:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764285.501592:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832709632/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.501597:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764285.501599:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832709632/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.501603:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764285.501605:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764285.501606:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764285.501606:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764285.501608:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764285.501617:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764285.501618:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764285.501620:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764285.501622:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764285.501625:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832709632/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764285.501630:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832709632:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764285.501633:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764285.501635:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832709760/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.501639:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764285.501640:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764285.501641:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764285.501642:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764285.501644:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764285.501646:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832709760:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764285.501648:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764285.501649:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764285.501650:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764285.501652:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764285.501654:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764285.501656:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764285.501658:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764285.501660:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764285.501661:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764285.501663:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832709760, portal 10
00000100:00000001:3.0:1787764285.501665:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764285.501666:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000100:00000040:3.0:1787764285.501670:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832709760/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764310 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.501674:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764285.501675:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764285.501677:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764285.501678:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832709760, offset 0
00000400:00000200:3.0:1787764285.501681:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764285.501683:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764285.501686:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764285.501688:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764285.501689:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764285.501691:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764285.501693:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764285.501695:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58780:58780:256:4294967295] 192.168.203.110@tcp LPNI seq info [58780:58780:8:4294967295]
00000400:00000200:3.0:1787764285.501698:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764285.501702:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764285.501705:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764285.501707:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764285.501709:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764285.501711:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787764285.501714:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764285.501716:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:3.0:1787764285.501718:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764285.501720:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764285.501722:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764285.501724:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764285.501726:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764285.501727:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764285.501729:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764285.501731:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e1c0 x1874599832709760/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764310 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.501736:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764285.501738:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832709760/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764310 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.501742:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764285.501743:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764285.501745:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764285.501747:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764285.501749:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764285.501751:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316672532
00000100:00000001:3.0:1787764285.501752:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316673779
00000100:00000001:3.0:1787764285.501754:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316672532
00000100:00000001:3.0:1787764285.501755:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316673779
00000100:00000001:3.0:1787764285.501756:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764285.501757:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764285.501759:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764285.501761:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764285/real 1787764285]  req@ffff9751f876e1c0 x1874599832709760/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764285.501765:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764285.501766:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764285.501767:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764285.501768:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764285.501769:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764285.501771:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764285.501773:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832709760/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764285.501778:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832709760/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.501782:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764285.501783:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764285.501784:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764285.501785:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764285.501787:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832709760/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.501791:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764285.501793:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832709760/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.501797:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764285.501799:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764285.501800:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764285.501801:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764285.501802:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764285.501803:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764285.501805:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764285.501807:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764285.501808:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764285.501810:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832709760/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764285.501815:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832709760:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764285.501817:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764285.501819:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764285.501820:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832709632/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.501824:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764285.501825:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764285.501826:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764285.501828:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764285.501829:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764285.501829:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764285.501831:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764285.501832:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:3.0:1787764285.501833:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764285.501834:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764285.501836:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:3.0:1787764285.501838:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764285.501839:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764285.501840:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764285.501842:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832709760/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.501855:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764285.501856:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764285.501858:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764285.501860:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764285.501862:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764285.501863:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764285.501865:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764285.501866:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016800.
02000000:00000001:3.0:1787764285.501869:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764285.501870:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764285.501872:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:3.0:1787764285.501875:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764285.501877:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764285.501879:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764285.501883:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764285.501885:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764285.501887:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764285.501889:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764285.501890:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764285.501895:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832709888/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.501900:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764285.501901:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764285.501902:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764285.501904:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764285.501905:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764285.501907:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832709888:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764285.501909:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764285.501911:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764285.501912:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764285.501913:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764285.501916:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764285.501917:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764285.501920:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764285.501922:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764285.501925:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764285.501927:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832709888, portal 4
00000100:00000001:3.0:1787764285.501928:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764285.501929:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00000040:3.0:1787764285.501933:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832709888/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764310 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.501937:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764285.501939:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764285.501940:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000200:3.0:1787764285.501942:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832709888, offset 0
00000400:00000200:3.0:1787764285.501944:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764285.501947:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764285.501949:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764285.501952:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764285.501954:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764285.501955:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764285.501957:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764285.501959:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58781:58781:256:4294967295] 192.168.203.110@tcp LPNI seq info [58781:58781:8:4294967295]
00000400:00000200:3.0:1787764285.501962:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764285.501966:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764285.501969:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764285.501972:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000800:00000200:3.0:1787764285.501974:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764285.501976:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000200:3.1:1787764285.501980:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764285.501982:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 1 secs later)
00000800:00000100:3.0:1787764285.501987:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764285.501989:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:3.0:1787764285.501992:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764285.501996:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764285.501999:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:3.0:1787764285.502001:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:3.0:1787764285.502004:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764285.502008:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e580 x1874599832709888/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764310 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.502017:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764285.502020:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832709888/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764310 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.502028:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764285.502030:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764285.502033:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764285.502036:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764285.502038:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764285.502039:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316672549
00000100:00000001:3.0:1787764285.502041:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316673779
00000100:00000001:3.0:1787764285.502043:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316672549
00000100:00000001:3.0:1787764285.502044:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316673779
00000100:00000001:3.0:1787764285.502046:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764285.502047:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764285.502049:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764285.502051:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764285/real 1787764285]  req@ffff9751f876e580 x1874599832709888/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764285.502056:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764285.502057:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764285.502058:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764285.502060:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764285.502061:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764285.502062:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764285.502064:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832709888/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764285.502070:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832709888/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.502074:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764285.502075:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764285.502076:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764285.502077:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764285.502080:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832709888/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.502084:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764285.502086:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832709888/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.502090:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764285.502091:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764285.502092:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764285.502093:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764285.502095:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764285.502096:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764285.502098:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764285.502100:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764285.502101:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764285.502104:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832709888/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764285.502108:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832709888:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764285.502111:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764285.502112:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764285.502114:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832709888/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764310 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764285.502117:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764285.502118:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764285.502120:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764285.502122:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764285.502123:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764285.502124:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764285.502125:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764285.502127:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014800.
02000000:00000001:3.0:1787764285.502128:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764285.502129:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764285.502131:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:3.0:1787764285.502153:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764285.502155:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764285.502158:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764285.502161:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764285.502163:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764285.502165:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764285.502166:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764286.140215:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764286.140222:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764286.140225:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764286.140228:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764286.140228:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764286.140230:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00100000:2.0:1787764286.140233:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764286.140235:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764286.140238:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764286.140238:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764286.140242:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764286.140245:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764286.140248:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764286.140248:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764286.140262:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764286.140262:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764286.140265:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764286.140270:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764286.140272:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764286.140275:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764286.140277:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764286.140320:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764286.140324:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764286.140326:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764286.524282:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764286.524290:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764286.524293:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764286.524299:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764286.524303:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764286.524308:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764286.524310:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764286.524313:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764286.524317:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764286.780304:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316673779, at 4316673792
00000100:00000010:3.0:1787764286.780321:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40f60.
00000100:00000040:2.0:1787764286.780357:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316673779, at 4316673792
00000100:00000010:2.0:1787764286.780372:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40660.
00000100:00000001:2.0:1787764287.164260:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764287.164265:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764287.164270:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764287.164271:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764287.164276:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764287.164277:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764287.164280:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764287.164283:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764287.164284:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764287.164286:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764287.164291:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764287.164298:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764287.164302:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764287.164305:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764287.164310:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764287.164363:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764287.164368:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764287.164371:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764287.548228:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764287.548236:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764287.548251:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764287.548259:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764287.548263:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764287.548268:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764287.548272:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764287.548275:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764287.548278:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764288.188271:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764288.188278:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764288.188279:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764288.188284:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764288.188287:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764288.188287:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764288.188292:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00100000:2.0:1787764288.188293:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764288.188296:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764288.188300:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764288.188304:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764288.188305:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764288.188307:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764288.188309:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764288.188309:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764288.188312:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764288.188314:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764288.188315:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764288.188326:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764288.188329:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764288.188333:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764288.188355:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764288.188360:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764288.188364:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764288.572278:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764288.572290:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764288.572293:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764288.572300:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764288.572305:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764288.572310:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764288.572313:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764288.572316:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764288.572329:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764289.212266:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764289.212270:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764289.212276:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764289.212279:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764289.212282:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764289.212284:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764289.212285:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764289.212292:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764289.212293:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764289.212294:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764289.212300:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764289.212304:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764289.212308:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764289.212312:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764289.212316:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764289.212354:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764289.212359:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764289.212362:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764289.596244:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764289.596251:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764289.596254:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764289.596260:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764289.596265:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764289.596269:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764289.596272:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764289.596275:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764289.596285:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764290.236249:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764290.236253:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764290.236257:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764290.236258:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764290.236264:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764290.236265:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764290.236269:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764290.236272:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764290.236274:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764290.236276:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764290.236279:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764290.236284:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764290.236287:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764290.236290:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764290.236294:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764290.236356:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764290.236364:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764290.236368:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764290.620212:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:2.0:1787764290.620215:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764290.620232:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:2.0:1787764290.620235:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764290.620235:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764290.620241:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:3.0:1787764290.620241:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:2.0:1787764290.620248:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764290.620248:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764290.620252:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764290.620254:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764290.620255:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764290.620259:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764290.620262:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764290.620266:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 22005
00000400:00000200:2.0:1787764290.620276:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764290.620284:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764290.620288:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764290.620292:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764290.620295:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764290.620298:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764290.620300:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764290.620304:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764290.620307:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787764290.620309:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764290.620313:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787764290.620315:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787764290.620318:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787764290.620321:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764290.620323:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764290.620326:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764290.620328:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764290.620331:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787764290.620335:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764290.620338:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764290.620341:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764290.620344:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764290.620347:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764290.620349:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764290.620352:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764290.620360:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000020:00000040:2.0:1787764290.620364:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787764290.620368:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764290.620370:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764290.620373:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764290.620375:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764290.620380:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764290.620388:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff975202074c00.
00000100:00000001:2.0:1787764290.620395:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764290.620404:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832710016/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764290.620415:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764290.620420:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764290.620425:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832710016/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764290.620447:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764290.620451:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764290.620455:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787764290.620457:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764290.620459:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764290.620463:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764290.620466:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00080000:2.0:1787764290.620467:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22005
00000100:00000001:3.0:1787764290.620470:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764290.620473:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000400:00000200:2.0:1787764290.620474:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764290.620482:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000040:3.0:1787764290.620482:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832710016/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787764290.620488:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764290.620495:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764290.620500:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764290.620502:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:2.0:1787764290.620505:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764290.620505:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764290.620512:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:3.0:1787764290.620512:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764290.620516:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:2.0:1787764290.620517:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764290.620520:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
02000000:00000001:3.0:1787764290.620522:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764290.620524:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764290.620528:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00100000:3.0:1787764290.620531:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832710016:192.168.203.110@tcp:250:kworker.0
00000020:00000001:2.0:1787764290.620532:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:2.0:1787764290.620538:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:3.0:1787764290.620540:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000040:2.0:1787764290.620545:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:3.0:1787764290.620547:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1787764290.620550:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1787764290.620551:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764290.620555:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764290.620558:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764290.620561:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:2.0:1787764290.620565:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:3.0:1787764290.620566:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1787764290.620570:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764290.620574:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000010:3.0:1787764290.620576:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:2.0:1787764290.620580:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764290.620581:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764290.620587:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764290.620592:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764290.620594:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:3.0:1787764290.620599:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000008:00100000:2.0:1787764290.620600:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787764290.620606:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764290.620609:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764290.620610:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000010:2.0:1787764290.620616:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000100:00000200:3.0:1787764290.620619:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832710016, portal 25
00000020:00000040:2.0:1787764290.620621:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000100:00000001:3.0:1787764290.620624:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000001:2.0:1787764290.620628:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1787764290.620628:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
02000000:00000001:2.0:1787764290.620632:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764290.620635:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000040:3.0:1787764290.620637:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832710016/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764315 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764290.620638:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764290.620642:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764290.620649:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000010:2.0:1787764290.620651:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff975202077800.
00000100:00000040:3.0:1787764290.620652:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764290.620657:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d323b8.
00000100:00000001:2.0:1787764290.620659:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764290.620661:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832710016, offset 0
00000100:00100000:2.0:1787764290.620670:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832710144/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787764290.620670:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764290.620681:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764290.620686:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1787764290.620690:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:2.0:1787764290.620693:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764290.620696:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:2.0:1787764290.620702:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832710144/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764290.620703:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764290.620708:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764290.620714:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764290.620721:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58782:58782:256:4294967295] 192.168.203.110@tcp LPNI seq info [58782:58782:8:4294967295]
00000100:00000001:2.0:1787764290.620728:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764290.620728:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:2.0:1787764290.620733:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1787764290.620738:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00080000:2.0:1787764290.620740:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1787764290.620744:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:3.0:1787764290.620745:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00080000:2.0:1787764290.620747:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000010:3.0:1787764290.620751:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186400.
00000100:00000001:2.0:1787764290.620752:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764290.620760:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22005
00000800:00000200:3.0:1787764290.620761:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000400:00000200:2.0:1787764290.620770:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:3.1:1787764290.620771:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000400:00000200:2.0:1787764290.620780:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764290.620784:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787764290.620787:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764290.620788:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764290.620794:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787764290.620795:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764290.620798:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316678899
00000100:00000040:2.0:1787764290.620800:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764290.620805:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000010:3.0:1787764290.620806:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40f60.
00000020:00000001:2.0:1787764290.620808:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764290.620812:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787764290.620814:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787764290.620816:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000020:00000040:2.0:1787764290.620818:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:2.0:1787764290.620823:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:2.0:1787764290.620826:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000001:3.0:1787764290.620828:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316678899
00000100:00000040:2.0:1787764290.620832:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764290.620836:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000010:3.0:1787764290.620837:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40cc0.
00000100:00000040:2.0:1787764290.620841:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764290.620844:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787764290.620845:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764290.620850:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000020:00000040:2.0:1787764290.620851:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00000001:3.0:1787764290.620857:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764290.620858:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764290.620865:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764290.620865:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764290.620872:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764290.620873:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:2.0:1787764290.620876:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764290.620881:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764290.620882:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764290.620883:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764290.620886:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000008:00100000:2.0:1787764290.620888:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787764290.620891:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000008:00000001:2.0:1787764290.620893:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764290.620895:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764290.620899:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764290.620900:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000010:2.0:1787764290.620908:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000100:00000040:3.0:1787764290.620909:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832710144/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787764290.620915:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764290.620922:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1787764290.620926:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1787764290.620927:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764290.620930:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764290.620931:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764290.620934:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764290.620935:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764290.620938:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764290.620939:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764290.620943:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764290.620947:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff975202077400.
00000100:00100000:3.0:1787764290.620948:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832710144:192.168.203.110@tcp:38:kworker.0
00000100:00000001:2.0:1787764290.620952:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764290.620954:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764290.620959:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764290.620962:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764290.620963:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832710272/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1787764290.620966:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764290.620973:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
00000100:00000001:2.0:1787764290.620975:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:3.0:1787764290.620978:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764290.620980:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764290.620986:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:2.0:1787764290.620988:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832710272/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1787764290.620992:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc234f20.
00000400:00000010:3.0:1787764290.620999:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000001:2.0:1787764290.621003:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764290.621005:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832710144, portal 10
00000100:00000040:2.0:1787764290.621008:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764290.621009:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764290.621013:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:2.0:1787764290.621016:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:2.0:1787764290.621021:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1787764290.621025:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832710144/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764315 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787764290.621026:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787764290.621030:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764290.621033:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764290.621038:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1787764290.621040:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:3.0:1787764290.621044:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00080000:2.0:1787764290.621048:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22005
00000400:00000010:3.0:1787764290.621049:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32a18.
00000100:00000200:3.0:1787764290.621054:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832710144, offset 0
00000400:00000200:2.0:1787764290.621057:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764290.621062:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764290.621066:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1787764290.621070:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:2.0:1787764290.621071:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1787764290.621076:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764290.621079:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764290.621081:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764290.621086:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000040:2.0:1787764290.621089:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000200:3.0:1787764290.621094:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:2.0:1787764290.621096:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764290.621102:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:3.0:1787764290.621104:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58783:58783:255:4294967295] 192.168.203.110@tcp LPNI seq info [58783:58783:7:4294967295]
00000100:00000001:2.0:1787764290.621106:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764290.621111:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764290.621115:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787764290.621118:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:3.0:1787764290.621118:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:2.0:1787764290.621122:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787764290.621126:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:2.0:1787764290.621132:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:2.0:1787764290.621158:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764290.621160:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787764290.621162:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000040:2.0:1787764290.621164:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764290.621169:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764290.621174:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000800:00000200:3.0:1787764290.621177:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00080000:2.0:1787764290.621181:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000800:00000010:3.0:1787764290.621184:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186800.
00000100:00000001:2.0:1787764290.621186:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764290.621190:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:3.0:1787764290.621195:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:2.0:1787764290.621196:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764290.621202:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787764290.621203:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787764290.621206:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764290.621211:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1787764290.621211:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000008:00000001:2.0:1787764290.621230:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764290.621233:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764290.621235:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000100:00000010:2.0:1787764290.621239:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
00000400:00000200:3.0:1787764290.621243:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:2.0:1787764290.621246:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:2.0:1787764290.621251:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1787764290.621252:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316677664
02000000:00000001:2.0:1787764290.621253:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764290.621257:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764290.621260:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1787764290.621261:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316678899
02000000:00000001:2.0:1787764290.621263:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764290.621268:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316677664
00000100:00000001:3.0:1787764290.621273:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316678899
02000000:00000010:2.0:1787764290.621275:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff975202074000.
00000100:00000001:3.0:1787764290.621280:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764290.621281:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764290.621286:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764290.621292:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832710400/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764290.621293:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764290.621298:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764290.621305:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764290.621308:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764290.621309:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:2.0:1787764290.621314:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764290.621315:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764290.621319:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764290.621322:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764290.621324:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832710400/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787764290.621330:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832710272/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764290.621345:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764290.621347:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764290.621351:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764290.621352:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (22015)
02000000:00000001:3.0:1787764290.621354:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764290.621359:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764290.621362:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764290.621366:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832710272:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764290.621369:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764290.621372:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764290.621374:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764290.621376:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764290.621380:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764290.621382:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764290.621386:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764290.621389:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc234268.
00000400:00000010:3.0:1787764290.621392:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32e58.
00000100:00000200:3.0:1787764290.621396:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832710272, portal 10
00000100:00000001:3.0:1787764290.621398:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764290.621399:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:3.0:1787764290.621406:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832710272/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764315 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764290.621413:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764290.621415:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764290.621418:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32770.
00000100:00000200:3.0:1787764290.621420:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832710272, offset 0
00000400:00000200:3.0:1787764290.621424:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764290.621428:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764290.621433:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764290.621435:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764290.621438:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764290.621440:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764290.621442:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764290.621445:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58784:58784:254:4294967295] 192.168.203.110@tcp LPNI seq info [58784:58784:6:4294967295]
00000400:00000200:3.0:1787764290.621449:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764290.621456:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764290.621461:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764290.621465:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186300.
00000800:00000200:3.0:1787764290.621469:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764290.621473:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787764290.621476:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764290.621480:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764290.621483:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764290.621485:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316677652
00000100:00000001:3.0:1787764290.621488:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316678899
00000100:00000001:3.0:1787764290.621490:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316677652
00000100:00000001:3.0:1787764290.621492:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316678899
00000100:00000001:3.0:1787764290.621494:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764290.621497:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764290.621499:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764290.621501:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764290.621504:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764290.621506:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1787764290.621508:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764290.621510:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764290.621511:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764290.621515:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832710400/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764290.621522:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764290.621523:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764290.621525:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764290.621527:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764290.621529:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764290.621531:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832710400:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764290.621534:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764290.621536:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764290.621538:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764290.621540:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764290.621543:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610c00.
02000000:00000001:3.0:1787764290.621544:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764290.621547:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764290.621550:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc234c60.
00000400:00000010:3.0:1787764290.621553:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d327f8.
00000100:00000200:3.0:1787764290.621556:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832710400, portal 4
00000100:00000001:3.0:1787764290.621558:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764290.621559:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:3.0:1787764290.621565:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832710400/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764315 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764290.621572:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764290.621574:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764290.621576:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d325d8.
00000100:00000200:3.0:1787764290.621578:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832710400, offset 0
00000400:00000200:3.0:1787764290.621583:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764290.621586:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764290.621590:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764290.621593:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764290.621596:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764290.621598:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764290.621600:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764290.621603:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58785:58785:253:4294967295] 192.168.203.110@tcp LPNI seq info [58785:58785:5:4294967295]
00000400:00000200:3.0:1787764290.621607:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764290.621613:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764290.621617:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764290.621621:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186500.
00000800:00000200:3.0:1787764290.621624:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764290.621628:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787764290.621631:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764290.621634:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764290.621637:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764290.621638:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316677669
00000100:00000001:3.0:1787764290.621641:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316678899
00000100:00000001:3.0:1787764290.621643:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316677669
00000100:00000001:3.0:1787764290.621645:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316678899
00000100:00000001:3.0:1787764290.621647:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764290.621650:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764290.621652:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764290.621655:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764290.621658:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764290.621659:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1787764290.621662:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764290.621663:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764290.621665:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764290.621666:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764291.260264:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764291.260272:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764291.260276:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764291.260278:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764291.260282:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764291.260284:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764291.260288:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764291.260289:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764291.260292:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764291.260296:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764291.260301:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764291.260305:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764291.260308:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764291.260311:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764291.260314:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764291.260322:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764291.260329:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764291.260334:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764291.644208:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764291.644226:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764291.644228:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764291.644234:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764291.644237:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764291.644241:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764291.644244:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764291.644246:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764291.644249:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764291.900274:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316678899, at 4316678912
00000100:00000010:2.0:1787764291.900292:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40f60.
00000100:00000040:3.0:1787764291.900295:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316678899, at 4316678912
00000100:00000010:3.0:1787764291.900321:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40cc0.
00010000:00000001:3.0:1787764292.284254:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764292.284260:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764292.284266:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764292.284267:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764292.284271:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764292.284273:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764292.284309:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764292.284318:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764292.284322:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764292.284325:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000100:00100000:2.0:1787764292.284329:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000020:00000001:0.0:1787764292.284330:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764292.284336:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764292.284338:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764292.284343:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764292.284348:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764292.284351:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764292.284355:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764292.668278:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764292.668289:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764292.668292:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764292.668299:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764292.668304:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764292.668310:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764292.668314:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764292.668317:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764292.668321:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764293.308242:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764293.308250:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764293.308251:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764293.308255:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764293.308257:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764293.308262:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764293.308266:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764293.308267:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764293.308271:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764293.308273:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764293.308277:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764293.308279:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764293.308280:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764293.308284:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764293.308294:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764293.308298:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764293.308302:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764293.308306:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764293.308308:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764293.308311:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764293.308314:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764293.308372:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764293.308378:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764293.308383:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764293.692245:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764293.692256:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764293.692258:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764293.692265:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764293.692270:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764293.692275:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764293.692279:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764293.692283:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764293.692287:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:2.0:1787764293.692493:0:257566:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.110:988
00000400:00000100:2.0:1787764293.692544:0:257566:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.110@tcp at host 192.168.203.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:2.0:1787764293.692555:0:257566:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764293.692560:0:257566:0:(router.c:1791:lnet_notify()) 192.168.203.10@tcp notifying 192.168.203.110@tcp: down
00000400:00000200:2.0:1787764293.692564:0:257566:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764293.692570:0:257566:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:2.0:1787764293.692576:0:257566:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:2.0:1787764293.692581:0:257566:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186400.
00000400:00000200:2.0:1787764293.692587:0:257566:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787764293.692593:0:257566:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787764293.692601:0:257566:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d323b8
00000400:00000010:2.0:1787764293.692605:0:257566:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d323b8.
00000100:00000001:2.0:1787764293.692611:0:257566:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764293.692622:0:257566:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f865ad00 x1874599832710016/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764315 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764293.692653:0:257566:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764293.692659:0:257566:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832710016/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764315 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764293.692669:0:257566:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764293.692671:0:257566:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787764293.692683:0:257566:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:2.0:1787764293.692686:0:257566:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186800.
00000400:00000200:2.0:1787764293.692689:0:257566:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787764293.692693:0:257566:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787764293.692697:0:257566:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32a18
00000400:00000010:2.0:1787764293.692699:0:257566:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32a18.
00000100:00000001:2.0:1787764293.692702:0:257566:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764293.692706:0:257566:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f86583c0 x1874599832710144/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764315 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764293.692715:0:257566:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764293.692719:0:257566:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832710144/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764315 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764293.692727:0:257566:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764293.692729:0:257566:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787764293.692733:0:257566:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:2.0:1787764293.692735:0:257566:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186300.
00000400:00000200:2.0:1787764293.692738:0:257566:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787764293.692741:0:257566:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787764293.692744:0:257566:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32770
00000400:00000010:2.0:1787764293.692746:0:257566:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32770.
00000100:00000001:2.0:1787764293.692749:0:257566:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764293.692753:0:257566:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f8659a40 x1874599832710272/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764315 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764293.692762:0:257566:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764293.692766:0:257566:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832710272/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764315 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764293.692772:0:257566:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764293.692774:0:257566:0:(events.c:68:request_out_callback()) Process leaving
00000800:00000100:2.0:1787764293.692778:0:257566:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:2.0:1787764293.692781:0:257566:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186500.
00000400:00000200:2.0:1787764293.692784:0:257566:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:2.0:1787764293.692787:0:257566:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:2.0:1787764293.692790:0:257566:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d325d8
00000400:00000010:2.0:1787764293.692793:0:257566:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d325d8.
00000100:00000001:2.0:1787764293.692795:0:257566:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:2.0:1787764293.692799:0:257566:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f8658000 x1874599832710400/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764315 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764293.692807:0:257566:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764293.692811:0:257566:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832710400/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764315 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764293.692818:0:257566:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764293.692820:0:257566:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764293.692884:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764293.692889:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764293.692892:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764293.692900:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764290/real 1787764293]  req@ffff9751f8658000 x1874599832710400/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764293.692913:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764293.692916:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764293.692920:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc234c60.
00000400:00000200:3.0:1787764293.692924:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d327f8
00000400:00000010:3.0:1787764293.692926:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d327f8.
00000100:00000001:3.0:1787764293.692929:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764293.692934:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832710400/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764293.692945:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832710400/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764293.692952:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764293.692955:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764293.692959:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764293.692961:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764293.692965:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832710400/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764293.692974:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764293.692980:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832710400/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764293.692990:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764293.692994:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764293.692996:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764293.692997:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764293.693001:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764293.693004:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764293.693008:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764293.693014:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764293.693017:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764293.693022:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832710400/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764293.693032:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832710400:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764293.693037:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764293.693041:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764290/real 1787764293]  req@ffff9751f8659a40 x1874599832710272/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764293.693049:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764293.693051:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764293.693053:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc234268.
00000400:00000200:3.0:1787764293.693055:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32e58
00000400:00000010:3.0:1787764293.693057:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32e58.
00000100:00000001:3.0:1787764293.693060:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764293.693063:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832710272/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764293.693073:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832710272/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764293.693079:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764293.693082:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764293.693084:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764293.693085:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764293.693090:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832710272/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764293.693097:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764293.693101:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832710272/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764293.693108:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764293.693111:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764293.693112:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764293.693114:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764293.693413:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764293.693416:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764293.693419:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764293.693424:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764293.693426:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764293.693432:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832710272/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764293.693442:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832710272:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764293.693447:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764293.693451:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764290/real 1787764293]  req@ffff9751f86583c0 x1874599832710144/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764293.693467:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764293.693468:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764293.693470:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc234f20.
00000400:00000200:3.0:1787764293.693473:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764293.693476:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764293.693479:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764293.693483:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832710144/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764293.693492:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832710144/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764293.693500:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764293.693501:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764293.693504:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764293.693505:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764293.693509:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832710144/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764293.693517:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764293.693521:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832710144/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764293.693528:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764293.693530:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764293.693532:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764293.693533:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764293.693535:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764293.693549:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764293.693551:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764293.693555:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764293.693558:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764293.693562:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832710144/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764293.693571:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832710144:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764293.693575:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764293.693580:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764290/real 1787764293]  req@ffff9751f865ad00 x1874599832710016/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764293.693589:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764293.693591:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764293.693593:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764293.693595:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764293.693598:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764293.693600:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764293.693604:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832710016/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764293.693618:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832710016/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764293.693625:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764293.693627:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764293.693629:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764293.693630:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764293.693634:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832710016/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764293.693642:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764293.693646:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832710016/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764293.693655:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764293.693657:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764293.693658:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764293.693659:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764293.693661:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764293.693663:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764293.693665:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764293.693670:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764293.693673:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764293.693677:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832710016/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764293.693690:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832710016:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764293.693695:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764293.693698:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764293.693703:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832710400/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764293.693710:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764293.693714:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764293.693719:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610c00.
02000000:00000001:3.0:1787764293.693723:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764293.693728:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764293.693730:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764293.693733:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764293.693737:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff975202074000.
02000000:00000001:3.0:1787764293.693746:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764293.693748:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764293.693750:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:3.0:1787764293.693754:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764293.693756:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764293.693758:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764293.693762:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832710272/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764293.693770:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764293.693772:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764293.693774:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764293.693776:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764293.693778:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764293.693779:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764293.693782:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764293.693784:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff975202077400.
02000000:00000001:3.0:1787764293.693786:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764293.693788:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764293.693790:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:3.0:1787764293.693793:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764293.693794:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764293.693796:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764293.693799:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832710144/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764293.693807:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764293.693808:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764293.693810:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764293.693812:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764293.693813:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764293.693815:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764293.693817:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764293.693818:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff975202077800.
02000000:00000001:3.0:1787764293.693821:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764293.693822:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764293.693823:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:3.0:1787764293.693826:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764293.693828:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764293.693829:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764293.693833:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832710016/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764315 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764293.693839:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764293.693840:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764293.693841:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764293.693844:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764293.693845:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764293.693846:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764293.693848:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764293.693850:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff975202074c00.
02000000:00000001:3.0:1787764293.693852:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764293.693853:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764293.693855:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:3.0:1787764293.693857:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764293.693858:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764293.693862:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764293.693871:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764293.693873:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764293.693876:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764293.693878:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764294.332275:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764294.332280:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764294.332294:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764294.332296:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764294.332298:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764294.332308:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764294.332309:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764294.332315:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764294.332316:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764294.332325:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764294.332325:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764294.332330:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764294.332332:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764294.332337:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764294.332341:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764294.332393:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764294.332396:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764294.332399:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764294.716211:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764294.716217:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764294.716219:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764294.716223:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764294.716227:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764294.716231:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764294.716233:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764294.716236:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764294.716239:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764295.356259:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764295.356262:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764295.356266:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764295.356267:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764295.356269:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764295.356270:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764295.356275:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764295.356279:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00100000:2.0:1787764295.356282:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764295.356283:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764295.356293:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764295.356294:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764295.356298:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764295.356299:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764295.356303:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764295.356307:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764295.356308:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764295.356312:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764295.356312:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764295.356322:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764295.356323:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764295.356378:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764295.356382:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764295.356386:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764295.740260:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764295.740268:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764295.740272:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764295.740278:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764295.740282:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764295.740286:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.740290:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764295.740293:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.740296:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764295.740357:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764295.740369:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787764295.740373:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764295.740378:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764295.740383:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764295.740394:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 22010
00000400:00000200:1.0:1787764295.740406:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764295.740415:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764295.740420:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764295.740424:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764295.740428:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764295.740431:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764295.740433:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764295.740439:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764295.740441:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787764295.740445:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764295.740450:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787764295.740453:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787764295.740456:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787764295.740460:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764295.740462:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764295.740464:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764295.740467:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764295.740470:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787764295.740475:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764295.740478:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764295.740482:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764295.740486:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764295.740490:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764295.740493:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764295.740497:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764295.740508:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930780.
00000020:00000040:1.0:1787764295.740513:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787764295.740518:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764295.740520:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764295.740524:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764295.740528:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764295.740533:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764295.740545:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb800.
00000100:00000001:1.0:1787764295.740554:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764295.740568:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930780 x1874599832710528/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764295.740581:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764295.740588:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764295.740595:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b3977950] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930780 x1874599832710528/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764295.740628:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764295.740633:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764295.740637:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787764295.740639:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764295.740640:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764295.740645:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764295.740648:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22010
00000400:00000200:1.0:1787764295.740652:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764295.740656:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764295.740659:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764295.740668:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764295.740672:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764295.740674:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764295.740677:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764295.740679:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764295.740681:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787764295.740682:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764295.740685:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787764295.740688:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787764295.740690:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787764295.740694:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764295.740696:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764295.740699:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764295.740701:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764295.740704:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787764295.740707:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764295.740710:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764295.740712:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764295.740719:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764295.740722:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764295.740724:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764295.740731:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787764295.740734:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764295.740737:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764295.740742:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932580.
00000020:00000040:1.0:1787764295.740745:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764295.740747:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764295.740748:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764295.740750:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764295.740752:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764295.740754:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764295.740758:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86ca000.
00000100:00000001:1.0:1787764295.740762:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764295.740768:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932580 x1874599832710656/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764295.740777:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764295.740779:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764295.740783:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008d026924] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932580 x1874599832710656/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764295.740793:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764295.740796:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764295.740800:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787764295.740801:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764295.740803:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764295.740806:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764295.740809:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22010
00000400:00000200:1.0:1787764295.740814:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764295.740817:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764295.740820:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764295.740824:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764295.740826:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764295.740828:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764295.740832:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764295.740834:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764295.740836:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787764295.740838:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764295.740840:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787764295.740842:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787764295.740845:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787764295.740848:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764295.740850:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764295.740852:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764295.740855:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764295.740857:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787764295.740861:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764295.740863:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764295.740865:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764295.740868:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764295.740870:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764295.740872:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764295.740875:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787764295.740878:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764295.740880:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764295.740884:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933840.
00000020:00000040:1.0:1787764295.740887:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764295.740890:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764295.740891:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764295.740894:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764295.740896:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764295.740898:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764295.740903:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9000.
00000100:00000001:1.0:1787764295.740908:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764295.740916:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933840 x1874599832710784/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764295.740917:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764295.740923:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764295.740929:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:1.0:1787764295.740943:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:3.0:1787764295.740946:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930780 x1874599832710528/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764295.740950:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764295.740986:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000083dd6d32] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933840 x1874599832710784/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.740986:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764295.740992:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764295.741004:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764295.741010:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764295.741014:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764295.741017:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764295.741021:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:3.0:1787764295.741028:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832710528:192.168.203.110@tcp:250:kworker.0
00000100:00000040:1.0:1787764295.741031:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:1.0:1787764295.741042:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764295.741043:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:1.0:1787764295.741051:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
02000000:00000001:3.0:1787764295.741052:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:1.0:1787764295.741063:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
02000000:00000001:3.0:1787764295.741064:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764295.741068:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
02000000:00000001:3.0:1787764295.741070:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:1.0:1787764295.741077:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764295.741084:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22010
02000000:00000010:3.0:1787764295.741090:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
00000400:00000200:1.0:1787764295.741091:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:3.0:1787764295.741101:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764295.741103:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764295.741110:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764295.741115:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764295.741117:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764295.741119:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764295.741122:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764295.741125:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764295.741126:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787764295.741128:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764295.741131:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:1.0:1787764295.741165:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787764295.741170:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:1.0:1787764295.741175:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764295.741178:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764295.741194:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764295.741198:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764295.741201:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:1.0:1787764295.741205:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764295.741208:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764295.741210:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764295.741215:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764295.741218:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764295.741221:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764295.741226:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:1.0:1787764295.741229:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764295.741232:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764295.741238:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7932940.
00000020:00000040:1.0:1787764295.741242:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:1.0:1787764295.741246:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764295.741247:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764295.741250:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764295.741251:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764295.741254:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764295.741261:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb400.
00000400:00000200:3.0:1787764295.741265:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764295.741268:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764295.741275:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000100:00100000:1.0:1787764295.741278:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7932940 x1874599832710912/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1787764295.741289:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000001:1.0:1787764295.741291:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000200:3.0:1787764295.741299:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832710528, portal 25
00000100:00000001:3.0:1787764295.741305:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764295.741307:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497723264 : -115097211828352 : ffff9751d7930780)
00000100:00000001:1.0:1787764295.741310:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764295.741335:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930780 x1874599832710528/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:1.0:1787764295.741336:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000af90eaa] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7932940 x1874599832710912/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764295.741416:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764295.741417:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764295.741429:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000040:1.0:1787764295.741431:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (22020)
00000400:00000010:3.0:1787764295.741434:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764295.741442:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832710528, offset 0
00000400:00000200:3.0:1787764295.741451:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764295.741462:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764295.741474:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764295.741480:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764295.741485:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764295.741490:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764295.741493:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764295.741497:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58786:58786:256:4294967295] 192.168.203.110@tcp LPNI seq info [58786:58786:8:4294967295]
00000400:00000200:3.0:1787764295.741501:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764295.741511:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764295.741518:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764295.741524:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764295.741532:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764295.741536:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787764295.741543:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764295.741546:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1787764295.741551:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764295.741554:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764295.741559:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764295.741563:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764295.741568:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764295.741570:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764295.741575:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764295.741580:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930780 x1874599832710528/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.741591:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764295.741595:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930780 x1874599832710528/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764320 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.741602:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764295.741604:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764295.741611:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764295.741614:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764295.741617:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316684020
00000100:00000010:3.0:1787764295.741623:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40cc0.
00000100:00000001:3.0:1787764295.741630:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787764295.741639:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316684020
00000100:00000010:3.0:1787764295.741642:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d409c0.
00000100:00000001:3.0:1787764295.741645:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787764295.741649:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764295.741664:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764295.741667:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764295.741672:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764295/real 1787764295]  req@ffff9751d7930780 x1874599832710528/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764295.741681:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764295.741683:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764295.741685:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764295.741688:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764295.741690:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764295.741693:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764295.741696:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930780 x1874599832710528/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764295.741704:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930780 x1874599832710528/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.741710:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764295.741712:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.741714:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764295.741716:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764295.741719:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930780 x1874599832710528/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.741726:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764295.741730:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930780 x1874599832710528/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.741738:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764295.741741:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.741743:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764295.741745:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.741747:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764295.741750:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764295.741753:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764295.741758:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764295.741761:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764295.741765:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930780 x1874599832710528/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764295.741774:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832710528:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764295.741779:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764295.741784:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932580 x1874599832710656/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.741791:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764295.741793:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764295.741796:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764295.741798:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764295.741801:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764295.741804:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832710656:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764295.741807:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764295.741810:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764295.741812:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764295.741814:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764295.741819:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764295.741821:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764295.741824:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764295.741827:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764295.741831:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764295.741834:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832710656, portal 10
00000100:00000001:3.0:1787764295.741836:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764295.741838:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497730944 : -115097211820672 : ffff9751d7932580)
00000100:00000040:3.0:1787764295.741844:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932580 x1874599832710656/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.741851:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764295.741854:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764295.741856:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764295.741858:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832710656, offset 0
00000400:00000200:3.0:1787764295.741862:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764295.741867:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764295.741871:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764295.741875:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764295.741878:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764295.741880:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764295.741882:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764295.741886:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58787:58787:256:4294967295] 192.168.203.110@tcp LPNI seq info [58787:58787:8:4294967295]
00000400:00000200:3.0:1787764295.741890:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764295.741896:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764295.741901:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764295.741905:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764295.741910:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764295.741913:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787764295.741919:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764295.741921:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1787764295.741925:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764295.741928:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764295.741931:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764295.741934:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764295.741938:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764295.741940:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764295.741942:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764295.741945:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932580 x1874599832710656/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.741950:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764295.741953:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932580 x1874599832710656/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764320 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.741959:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764295.741961:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764295.741965:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764295.741973:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764295.741977:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764295.741980:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316682784
00000100:00000001:3.0:1787764295.741986:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316684020
00000100:00000001:3.0:1787764295.741988:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316682784
00000100:00000001:3.0:1787764295.741989:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316684020
00000100:00000001:3.0:1787764295.741992:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764295.741994:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764295.741996:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764295.742000:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764295/real 1787764295]  req@ffff9751d7932580 x1874599832710656/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764295.742008:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764295.742009:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764295.742011:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764295.742014:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764295.742015:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764295.742018:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764295.742020:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932580 x1874599832710656/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764295.742028:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932580 x1874599832710656/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742035:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764295.742037:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.742039:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764295.742040:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764295.742044:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932580 x1874599832710656/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742051:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764295.742055:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932580 x1874599832710656/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742062:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764295.742065:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.742066:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764295.742067:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.742069:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764295.742083:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764295.742086:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764295.742090:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764295.742092:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764295.742096:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932580 x1874599832710656/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764295.742103:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832710656:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764295.742107:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.742110:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764295.742113:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930780 x1874599832710528/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742119:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764295.742121:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764295.742124:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764295.742127:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764295.742130:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764295.742132:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764295.742163:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764295.742166:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb800.
02000000:00000001:3.0:1787764295.742171:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764295.742173:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764295.742175:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930780.
00000100:00000001:3.0:1787764295.742179:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764295.742180:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.742193:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764295.742197:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932580 x1874599832710656/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742204:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764295.742206:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764295.742207:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764295.742210:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764295.742211:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764295.742213:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764295.742214:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764295.742216:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86ca000.
02000000:00000001:3.0:1787764295.742218:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764295.742220:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764295.742221:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932580.
00000100:00000001:3.0:1787764295.742224:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764295.742225:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.742228:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.742237:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764295.742240:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764295.742243:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764295.742245:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764295.742246:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764295.742252:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933840 x1874599832710784/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742260:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764295.742262:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764295.742264:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764295.742266:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764295.742268:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764295.742271:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832710784:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764295.742275:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764295.742278:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764295.742280:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764295.742282:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764295.742286:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764295.742288:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764295.742292:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764295.742295:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764295.742298:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764295.742302:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832710784, portal 10
00000100:00000001:3.0:1787764295.742303:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764295.742305:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497735744 : -115097211815872 : ffff9751d7933840)
00000100:00000040:3.0:1787764295.742311:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933840 x1874599832710784/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742318:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764295.742321:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764295.742324:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764295.742326:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832710784, offset 0
00000400:00000200:3.0:1787764295.742331:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764295.742336:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764295.742341:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764295.742344:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764295.742347:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764295.742350:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764295.742352:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764295.742355:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58788:58788:256:4294967295] 192.168.203.110@tcp LPNI seq info [58788:58788:8:4294967295]
00000400:00000200:3.0:1787764295.742359:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764295.742366:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764295.742370:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764295.742374:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764295.742379:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764295.742382:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787764295.742386:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764295.742388:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1787764295.742391:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764295.742392:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764295.742394:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764295.742397:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764295.742399:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764295.742401:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764295.742402:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764295.742405:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933840 x1874599832710784/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742411:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764295.742413:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933840 x1874599832710784/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764320 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742417:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764295.742418:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764295.742421:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764295.742423:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764295.742425:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764295.742427:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316682773
00000100:00000001:3.0:1787764295.742429:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316684020
00000100:00000001:3.0:1787764295.742430:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316682773
00000100:00000001:3.0:1787764295.742432:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316684020
00000100:00000001:3.0:1787764295.742433:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764295.742434:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764295.742435:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764295.742438:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764295/real 1787764295]  req@ffff9751d7933840 x1874599832710784/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764295.742443:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764295.742444:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764295.742446:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764295.742448:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764295.742450:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764295.742452:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764295.742456:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933840 x1874599832710784/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764295.742464:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933840 x1874599832710784/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742471:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764295.742472:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.742474:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764295.742476:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764295.742480:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933840 x1874599832710784/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742487:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764295.742491:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933840 x1874599832710784/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742497:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764295.742500:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.742501:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764295.742502:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.742504:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764295.742505:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764295.742508:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764295.742511:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764295.742514:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764295.742518:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933840 x1874599832710784/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764295.742526:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832710784:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764295.742531:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764295.742535:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7932940 x1874599832710912/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742542:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764295.742544:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764295.742546:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764295.742548:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764295.742550:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764295.742553:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832710912:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764295.742556:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764295.742558:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764295.742560:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764295.742562:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764295.742565:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764295.742567:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764295.742571:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764295.742574:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764295.742577:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764295.742580:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832710912, portal 4
00000100:00000001:3.0:1787764295.742582:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764295.742583:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497731904 : -115097211819712 : ffff9751d7932940)
00000100:00000040:3.0:1787764295.742589:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7932940 x1874599832710912/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742596:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764295.742598:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764295.742601:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764295.742604:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832710912, offset 0
00000400:00000200:3.0:1787764295.742608:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764295.742612:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764295.742616:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764295.742619:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764295.742622:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764295.742625:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764295.742627:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764295.742630:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58789:58789:256:4294967295] 192.168.203.110@tcp LPNI seq info [58789:58789:8:4294967295]
00000400:00000200:3.0:1787764295.742633:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764295.742638:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764295.742643:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764295.742647:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764295.742651:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764295.742654:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000200:3.1:1787764295.742658:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764295.742661:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 58 secs later)
00000800:00000100:3.0:1787764295.742666:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764295.742668:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764295.742671:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764295.742674:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764295.742677:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764295.742679:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764295.742682:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764295.742687:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7932940 x1874599832710912/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764320 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742694:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764295.742697:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7932940 x1874599832710912/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764320 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742703:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764295.742705:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764295.742708:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764295.742711:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764295.742714:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764295.742716:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316682790
00000100:00000001:3.0:1787764295.742719:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316684020
00000100:00000001:3.0:1787764295.742721:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316682790
00000100:00000001:3.0:1787764295.742724:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316684020
00000100:00000001:3.0:1787764295.742726:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764295.742729:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764295.742731:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764295.742735:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764295/real 1787764295]  req@ffff9751d7932940 x1874599832710912/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764295.742740:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764295.742741:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764295.742742:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764295.742744:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764295.742745:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764295.742747:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764295.742749:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7932940 x1874599832710912/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764295.742755:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7932940 x1874599832710912/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742758:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764295.742759:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.742761:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764295.742761:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764295.742764:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7932940 x1874599832710912/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742768:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764295.742770:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7932940 x1874599832710912/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742775:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764295.742776:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.742777:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764295.742778:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.742780:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764295.742781:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764295.742783:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764295.742785:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764295.742786:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764295.742789:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7932940 x1874599832710912/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764295.742794:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7932940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832710912:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764295.742796:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.742797:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764295.742799:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933840 x1874599832710784/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742803:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764295.742805:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764295.742806:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764295.742808:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764295.742809:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764295.742810:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764295.742812:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764295.742813:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9000.
02000000:00000001:3.0:1787764295.742815:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764295.742816:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764295.742817:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933840.
00000100:00000001:3.0:1787764295.742820:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764295.742821:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.742822:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764295.742824:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7932940 x1874599832710912/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764320 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764295.742828:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764295.742828:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764295.742829:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764295.742831:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764295.742832:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764295.742832:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764295.742834:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764295.742835:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb400.
02000000:00000001:3.0:1787764295.742836:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764295.742837:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764295.742838:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7932940.
00000100:00000001:3.0:1787764295.742839:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764295.742840:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.742841:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764295.742843:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764295.742844:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764295.742846:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764295.742847:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764296.380250:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764296.380260:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764296.380263:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764296.380268:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764296.380273:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764296.380277:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764296.380280:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764296.380283:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764296.380286:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764296.380344:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764296.380347:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764296.380351:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764296.380352:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764296.380354:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764296.380355:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764296.380436:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764296.380443:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764296.380450:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764296.764242:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764296.764248:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764296.764250:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764296.764255:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764296.764258:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764296.764261:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764296.764263:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764296.764265:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764296.764267:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764297.020253:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316684020, at 4316684032
00000100:00000010:3.0:1787764297.020267:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d409c0.
00000100:00000040:2.0:1787764297.020286:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316684020, at 4316684032
00000100:00000010:2.0:1787764297.020297:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40cc0.
00000100:00000001:2.0:1787764297.404240:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764297.404250:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764297.404253:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764297.404260:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764297.404264:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764297.404268:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764297.404272:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764297.404275:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764297.404279:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764297.404286:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764297.404294:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764297.404302:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764297.404306:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764297.404309:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764297.404313:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764297.404317:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764297.404320:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764297.404322:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764297.404325:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764297.404329:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764297.404330:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764297.404386:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764297.404391:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764297.404395:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764297.788260:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764297.788268:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764297.788270:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764297.788276:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764297.788279:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764297.788282:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764297.788285:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764297.788287:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764297.788290:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764298.428303:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764298.428304:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764298.428310:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764298.428311:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764298.428314:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764298.428317:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764298.428317:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764298.428319:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764298.428323:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764298.428325:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764298.428327:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764298.428327:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764298.428330:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764298.428333:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764298.428337:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764298.428360:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764298.428366:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764298.428370:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764298.812300:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764298.812310:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764298.812312:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764298.812321:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764298.812327:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764298.812333:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764298.812337:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764298.812341:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764298.812346:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764299.452250:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764299.452253:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764299.452257:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764299.452258:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764299.452260:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764299.452264:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764299.452265:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764299.452267:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764299.452269:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764299.452271:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764299.452273:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764299.452274:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764299.452279:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764299.452283:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764299.452285:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764299.452349:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764299.452355:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764299.452359:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764299.836296:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764299.836306:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764299.836308:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764299.836315:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764299.836320:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764299.836325:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764299.836328:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764299.836331:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764299.836334:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764300.476251:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764300.476258:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764300.476259:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764300.476262:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764300.476264:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764300.476269:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764300.476270:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764300.476272:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764300.476278:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764300.476279:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764300.476285:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764300.476291:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764300.476295:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764300.476298:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764300.476301:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764300.476319:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764300.476322:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764300.476325:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764300.860244:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764300.860253:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764300.860256:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764300.860262:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:2.0:1787764300.860266:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787764300.860267:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764300.860273:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1787764300.860274:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787764300.860278:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:1.0:1787764300.860279:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:2.0:1787764300.860282:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764300.860285:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764300.860286:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1787764300.860292:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764300.860293:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 22015
00000400:00000200:2.0:1787764300.860302:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764300.860307:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764300.860309:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764300.860313:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764300.860315:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764300.860317:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764300.860319:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764300.860322:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764300.860323:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787764300.860326:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764300.860329:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787764300.860330:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787764300.860332:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787764300.860334:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764300.860336:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764300.860337:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764300.860338:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764300.860340:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787764300.860342:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764300.860344:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764300.860346:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764300.860348:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764300.860351:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764300.860352:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764300.860354:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764300.860361:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658780.
00000020:00000040:2.0:1787764300.860363:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787764300.860366:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764300.860367:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764300.860369:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764300.860371:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764300.860374:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764300.860380:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f6800.
00000100:00000001:2.0:1787764300.860386:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764300.860393:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658780 x1874599832711040/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764300.860400:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764300.860403:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764300.860407:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000744875b5] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658780 x1874599832711040/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764300.860427:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764300.860429:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764300.860431:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787764300.860432:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764300.860434:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764300.860436:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764300.860438:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22015
00000400:00000200:2.0:1787764300.860440:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764300.860442:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764300.860443:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764300.860445:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764300.860447:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764300.860449:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764300.860450:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764300.860452:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764300.860453:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:2.0:1787764300.860455:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764300.860456:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:2.0:1787764300.860458:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:2.0:1787764300.860460:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:2.0:1787764300.860462:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764300.860464:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764300.860466:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764300.860468:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764300.860468:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1787764300.860473:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00000001:3.0:1787764300.860475:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:2.0:1787764300.860476:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764300.860480:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764300.860480:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
02000000:00000001:2.0:1787764300.860484:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764300.860490:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764300.860492:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764300.860492:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832711040/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764300.860496:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764300.860501:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787764300.860505:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764300.860509:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764300.860514:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000010:2.0:1787764300.860516:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000100:00000001:3.0:1787764300.860517:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1787764300.860521:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787764300.860524:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:2.0:1787764300.860525:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764300.860528:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764300.860530:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764300.860531:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:3.0:1787764300.860531:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764300.860535:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764300.860537:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764300.860539:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f7400.
00000100:00000001:2.0:1787764300.860544:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764300.860550:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832711040:192.168.203.110@tcp:250:kworker.0
00000100:00100000:2.0:1787764300.860551:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832711168/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764300.860563:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1787764300.860564:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764300.860568:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764300.860571:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:2.0:1787764300.860572:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832711168/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:3.0:1787764300.860576:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764300.860580:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764300.860583:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00080000:2.0:1787764300.860584:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764300.860587:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1787764300.860588:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764300.860590:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
02000000:00000010:3.0:1787764300.860592:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
00000100:00000001:2.0:1787764300.860593:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764300.860597:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22015
02000000:00000001:3.0:1787764300.860598:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764300.860602:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764300.860605:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764300.860606:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1787764300.860609:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:2.0:1787764300.860611:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764300.860614:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000010:3.0:1787764300.860616:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000100:00000040:2.0:1787764300.860617:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764300.860621:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764300.860624:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764300.860625:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787764300.860626:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764300.860627:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000400:00000010:3.0:1787764300.860628:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000020:00000001:2.0:1787764300.860629:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:2.0:1787764300.860634:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:2.0:1787764300.860636:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764300.860638:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:3.0:1787764300.860639:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832711040, portal 25
00000100:00000040:2.0:1787764300.860640:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764300.860643:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764300.860645:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000020:00000040:2.0:1787764300.860646:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:2.0:1787764300.860650:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764300.860650:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000001:2.0:1787764300.860654:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764300.860657:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764300.860659:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764300.860662:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764300.860664:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:3.0:1787764300.860664:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832711040/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:2.0:1787764300.860667:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787764300.860671:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764300.860674:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764300.860681:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000020:00000040:2.0:1787764300.860685:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000001:3.0:1787764300.860685:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:2.0:1787764300.860687:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764300.860689:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764300.860693:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787764300.860694:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764300.860696:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764300.860699:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764300.860701:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
02000000:00000010:2.0:1787764300.860703:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f6c00.
00000100:00000001:2.0:1787764300.860707:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764300.860707:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832711040, offset 0
00000100:00100000:2.0:1787764300.860714:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832711296/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787764300.860722:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787764300.860724:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764300.860727:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764300.860730:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832711296/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764300.860736:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764300.860742:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764300.860744:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1787764300.860747:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:3.0:1787764300.860749:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00080000:2.0:1787764300.860750:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764300.860757:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787764300.860759:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1787764300.860760:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:2.0:1787764300.860761:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764300.860766:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1787764300.860770:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00080000:2.0:1787764300.860773:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22015
00000400:00000200:3.0:1787764300.860778:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764300.860782:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764300.860787:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58790:58790:256:4294967295] 192.168.203.110@tcp LPNI seq info [58790:58790:8:4294967295]
00000400:00000200:3.0:1787764300.860792:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764300.860801:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764300.860810:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764300.860811:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764300.860818:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000800:00000010:3.0:1787764300.860818:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000100:00080000:2.0:1787764300.860824:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764300.860827:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764300.860831:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1787764300.860833:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:2.0:1787764300.860836:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764300.860839:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:3.0:1787764300.860840:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000020:00000001:2.0:1787764300.860841:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764300.860844:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787764300.860847:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764300.860850:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787764300.860853:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000800:00000200:3.1:1787764300.860855:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:2.0:1787764300.860856:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:2.0:1787764300.860863:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:3.1:1787764300.860863:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000100:00000001:2.0:1787764300.860867:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000100:3.0:1787764300.860875:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000040:2.0:1787764300.860878:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000010:3.0:1787764300.860880:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000100:00000001:2.0:1787764300.860881:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764300.860885:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000400:00000200:3.0:1787764300.860888:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:2.0:1787764300.860889:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764300.860895:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764300.860897:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764300.860901:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1787764300.860901:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:2.0:1787764300.860905:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764300.860908:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764300.860910:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000400:00000200:3.0:1787764300.860910:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000008:00000001:2.0:1787764300.860916:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764300.860916:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:2.0:1787764300.860918:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764300.860923:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000010:2.0:1787764300.860925:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000020:00000040:2.0:1787764300.860931:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000200:3.0:1787764300.860932:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832711040/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764300.860937:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764300.860940:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764300.860945:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764300.860948:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764300.860952:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764300.860953:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000010:2.0:1787764300.860958:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f6400.
00000100:00000040:3.0:1787764300.860960:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832711040/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764325 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764300.860965:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764300.860974:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832711424/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764300.860977:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764300.860982:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764300.860987:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764300.860990:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764300.860990:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764300.860997:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832711424/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764300.861001:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764300.861007:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316689139
00000100:00000001:2.0:1787764300.861009:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764300.861014:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (22025)
00000100:00000010:3.0:1787764300.861015:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d409c0.
00000100:00000001:3.0:1787764300.861024:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787764300.861032:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316689139
00000100:00000010:3.0:1787764300.861035:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d404e0.
00000100:00000001:3.0:1787764300.861039:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787764300.861042:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764300.861045:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764300.861048:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764300.861054:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764300/real 1787764300]  req@ffff9751f8658780 x1874599832711040/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764300.861063:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764300.861065:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764300.861067:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764300.861069:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764300.861071:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764300.861073:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764300.861077:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832711040/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764300.861087:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832711040/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.861095:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764300.861096:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764300.861098:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764300.861101:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764300.861105:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832711040/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.861114:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764300.861119:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832711040/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.861126:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764300.861129:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764300.861131:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764300.861132:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764300.861170:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764300.861172:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764300.861175:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764300.861180:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764300.861183:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764300.861188:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832711040/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764300.861197:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832711040:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764300.861202:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764300.861204:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764300.861208:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832711040/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.861215:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764300.861217:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764300.861220:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764300.861223:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764300.861225:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764300.861227:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764300.861229:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764300.861232:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f6800.
02000000:00000001:3.0:1787764300.861236:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764300.861238:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764300.861240:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:3.0:1787764300.861243:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764300.861244:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764300.861246:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764300.861252:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764300.861253:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764300.861256:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764300.861257:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764300.861259:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764300.861262:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832711168/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.861270:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764300.861272:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764300.861274:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764300.861276:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764300.861279:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764300.861282:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832711168:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764300.861285:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764300.861287:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764300.861289:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764300.861291:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764300.861294:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764300.861296:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764300.861300:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764300.861303:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764300.861307:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764300.861309:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832711168, portal 10
00000100:00000001:3.0:1787764300.861311:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764300.861313:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:3.0:1787764300.861319:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832711168/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.861327:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764300.861329:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764300.861331:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764300.861334:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832711168, offset 0
00000400:00000200:3.0:1787764300.861338:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764300.861342:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764300.861347:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764300.861350:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764300.861352:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764300.861355:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764300.861358:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764300.861362:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58791:58791:256:4294967295] 192.168.203.110@tcp LPNI seq info [58791:58791:8:4294967295]
00000400:00000200:3.0:1787764300.861366:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764300.861374:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764300.861378:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764300.861382:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764300.861386:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764300.861389:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787764300.861394:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764300.861397:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1787764300.861401:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764300.861403:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764300.861406:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764300.861410:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764300.861413:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764300.861415:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764300.861417:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764300.861421:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832711168/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.861431:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764300.861434:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832711168/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764325 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.861442:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764300.861444:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764300.861447:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764300.861450:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764300.861454:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764300.861456:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316687904
00000100:00000001:3.0:1787764300.861459:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316689139
00000100:00000001:3.0:1787764300.861461:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316687904
00000100:00000001:3.0:1787764300.861463:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316689139
00000100:00000001:3.0:1787764300.861465:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764300.861468:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764300.861470:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764300.861474:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764300/real 1787764300]  req@ffff9751f865ad00 x1874599832711168/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764300.861489:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764300.861491:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764300.861493:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764300.861495:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764300.861497:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764300.861499:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764300.861502:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832711168/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764300.861512:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832711168/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.861519:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764300.861521:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764300.861523:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764300.861524:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764300.861528:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832711168/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.861535:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764300.861540:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832711168/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.861547:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764300.861550:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764300.861551:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764300.861553:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764300.861555:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764300.861567:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764300.861570:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764300.861574:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764300.861576:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764300.861581:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832711168/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764300.861589:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832711168:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764300.861593:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764300.861597:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832711296/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.861605:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764300.861607:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764300.861609:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764300.861611:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764300.861613:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764300.861615:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832711296:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764300.861619:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764300.861621:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764300.861622:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764300.861624:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764300.861628:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764300.861630:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764300.861633:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764300.861636:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764300.861639:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764300.861642:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832711296, portal 10
00000100:00000001:3.0:1787764300.861644:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764300.861645:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:3.0:1787764300.861651:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832711296/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.861658:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764300.861661:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764300.861663:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764300.861665:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832711296, offset 0
00000400:00000200:3.0:1787764300.861670:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764300.861674:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764300.861677:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764300.861680:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764300.861682:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764300.861685:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764300.861688:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764300.861691:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58792:58792:256:4294967295] 192.168.203.110@tcp LPNI seq info [58792:58792:8:4294967295]
00000400:00000200:3.0:1787764300.861696:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764300.861702:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764300.861707:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764300.861710:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764300.861714:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764300.861717:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787764300.861722:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764300.861725:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1787764300.861729:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764300.861731:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764300.861734:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764300.861738:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764300.861741:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764300.861743:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764300.861745:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764300.861749:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832711296/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.861757:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764300.861761:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832711296/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764325 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.861767:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764300.861769:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764300.861772:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764300.861775:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764300.861778:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764300.861780:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316687892
00000100:00000001:3.0:1787764300.861783:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316689139
00000100:00000001:3.0:1787764300.861785:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316687892
00000100:00000001:3.0:1787764300.861787:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316689139
00000100:00000001:3.0:1787764300.861790:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764300.861792:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764300.861794:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764300.861798:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764300/real 1787764300]  req@ffff9751f86583c0 x1874599832711296/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764300.861805:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764300.861807:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764300.861809:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764300.861811:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764300.861813:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764300.861815:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764300.861819:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832711296/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764300.861828:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832711296/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.861833:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764300.861835:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764300.861837:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764300.861838:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764300.861843:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832711296/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.861850:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764300.861854:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832711296/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.861862:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764300.861866:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764300.861867:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764300.861869:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764300.861870:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764300.861872:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764300.861875:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764300.861879:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764300.861881:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764300.861885:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832711296/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764300.861894:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832711296:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764300.861898:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764300.861901:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832711424/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.861908:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764300.861910:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764300.861912:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764300.861914:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764300.861916:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764300.861920:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832711424:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764300.861923:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764300.861925:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764300.861926:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764300.861928:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764300.861932:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764300.861933:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764300.861937:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764300.861939:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764300.861942:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764300.861945:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832711424, portal 4
00000100:00000001:3.0:1787764300.861948:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764300.861950:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:3.0:1787764300.861954:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832711424/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.861962:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764300.861964:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764300.861966:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764300.861969:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832711424, offset 0
00000400:00000200:3.0:1787764300.861973:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764300.861976:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764300.861980:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764300.861982:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764300.861985:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764300.861988:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764300.861990:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764300.861994:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58793:58793:256:4294967295] 192.168.203.110@tcp LPNI seq info [58793:58793:8:4294967295]
00000400:00000200:3.0:1787764300.861998:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764300.862004:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764300.862009:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764300.862012:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764300.862016:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764300.862019:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000200:3.1:1787764300.862023:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764300.862026:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 53 secs later)
00000800:00000100:3.0:1787764300.862030:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764300.862032:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764300.862035:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764300.862039:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764300.862042:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764300.862044:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764300.862046:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764300.862050:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832711424/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764325 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.862064:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764300.862067:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832711424/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764325 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.862075:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764300.862076:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764300.862079:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764300.862082:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764300.862085:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764300.862087:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316687909
00000100:00000001:3.0:1787764300.862090:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316689139
00000100:00000001:3.0:1787764300.862092:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316687909
00000100:00000001:3.0:1787764300.862094:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316689139
00000100:00000001:3.0:1787764300.862096:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764300.862098:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764300.862100:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764300.862104:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764300/real 1787764300]  req@ffff9751f8659a40 x1874599832711424/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764300.862112:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764300.862113:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764300.862114:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764300.862116:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764300.862118:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764300.862120:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764300.862124:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832711424/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764300.862161:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832711424/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.862170:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764300.862172:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764300.862174:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764300.862175:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764300.862180:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832711424/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.862188:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764300.862192:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832711424/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.862199:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764300.862202:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764300.862204:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764300.862205:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764300.862207:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764300.862208:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764300.862211:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764300.862214:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764300.862217:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764300.862221:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832711424/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764300.862228:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832711424:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764300.862232:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764300.862234:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764300.862237:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832711168/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.862244:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764300.862246:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764300.862248:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764300.862251:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764300.862252:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764300.862254:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764300.862256:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764300.862258:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f7400.
02000000:00000001:3.0:1787764300.862261:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764300.862262:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764300.862263:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:3.0:1787764300.862267:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764300.862268:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764300.862270:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764300.862273:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832711296/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.862280:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764300.862282:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764300.862283:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764300.862285:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764300.862287:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764300.862288:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764300.862290:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764300.862291:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f6c00.
02000000:00000001:3.0:1787764300.862294:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764300.862295:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764300.862297:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:3.0:1787764300.862299:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764300.862300:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764300.862302:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764300.862305:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832711424/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764325 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764300.862311:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764300.862313:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764300.862314:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764300.862316:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764300.862317:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764300.862319:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764300.862321:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764300.862322:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f6400.
02000000:00000001:3.0:1787764300.862325:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764300.862326:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764300.862328:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:3.0:1787764300.862330:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764300.862331:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764300.862333:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764300.862337:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764300.862339:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764300.862341:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764300.862342:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764301.500228:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764301.500232:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764301.500238:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764301.500239:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764301.500243:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764301.500246:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764301.500249:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764301.500250:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764301.500257:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764301.500263:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764301.500265:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764301.500267:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764301.500269:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764301.500271:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764301.500277:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764301.500313:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764301.500317:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764301.500321:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764301.884253:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764301.884260:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764301.884262:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764301.884266:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764301.884270:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764301.884273:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764301.884275:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764301.884278:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764301.884280:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764302.140238:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316689139, at 4316689152
00000100:00000010:3.0:1787764302.140252:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d404e0.
00000100:00000040:2.0:1787764302.140286:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316689139, at 4316689152
00000100:00000010:2.0:1787764302.140296:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d409c0.
00000100:00000001:2.0:1787764302.524252:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764302.524257:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764302.524261:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764302.524262:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764302.524266:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764302.524272:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764302.524272:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764302.524276:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764302.524278:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764302.524283:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764302.524284:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764302.524287:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764302.524289:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764302.524293:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764302.524296:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764302.524359:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764302.524364:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764302.524368:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764302.908242:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764302.908249:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764302.908252:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764302.908256:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764302.908259:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764302.908262:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764302.908264:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764302.908266:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764302.908268:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764303.548216:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764303.548220:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764303.548223:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764303.548225:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764303.548226:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764303.548233:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764303.548236:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764303.548237:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764303.548241:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764303.548242:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764303.548247:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764303.548249:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764303.548251:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764303.548253:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764303.548269:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764303.548272:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764303.548276:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764303.548279:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764303.548281:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764303.548283:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764303.548284:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764303.548328:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764303.548330:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764303.548333:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764303.932271:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764303.932280:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764303.932283:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764303.932288:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764303.932294:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764303.932297:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764303.932300:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764303.932303:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764303.932306:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764304.572243:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764304.572246:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764304.572250:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764304.572253:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764304.572256:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764304.572258:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764304.572259:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764304.572265:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764304.572267:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764304.572267:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764304.572273:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764304.572277:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764304.572281:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764304.572283:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764304.572287:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764304.572309:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764304.572314:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764304.572317:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764304.956216:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764304.956225:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764304.956228:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764304.956233:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764304.956237:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764304.956241:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764304.956243:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764304.956246:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764304.956250:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764305.596224:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764305.596224:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764305.596233:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764305.596233:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764305.596236:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764305.596237:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764305.596242:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764305.596244:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764305.596247:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764305.596250:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764305.596251:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764305.596255:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764305.596256:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764305.596259:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764305.596261:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764305.596265:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764305.596265:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764305.596269:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764305.596271:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764305.596275:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764305.596278:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764305.596313:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764305.596316:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764305.596319:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787764305.980250:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764305.980258:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787764305.980261:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764305.980264:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764305.980269:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764305.980269:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:2.0:1787764305.980276:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 22020
00000100:00000001:3.0:1787764305.980276:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764305.980281:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764305.980286:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00100000:3.0:1787764305.980286:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764305.980293:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000400:00000200:2.0:1787764305.980297:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787764305.980299:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1787764305.980300:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787764305.980304:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:2.0:1787764305.980305:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764305.980307:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764305.980310:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764305.980312:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764305.980315:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764305.980318:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764305.980321:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764305.980322:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787764305.980325:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764305.980328:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787764305.980330:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787764305.980332:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787764305.980334:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764305.980335:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764305.980337:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764305.980339:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764305.980341:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787764305.980344:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764305.980346:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764305.980348:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764305.980351:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764305.980353:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764305.980355:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764305.980356:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764305.980364:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
00000020:00000040:2.0:1787764305.980366:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787764305.980369:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764305.980370:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764305.980373:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764305.980375:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764305.980378:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764305.980385:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f4800.
00000100:00000001:2.0:1787764305.980390:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764305.980397:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832711552/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764305.980405:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764305.980408:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764305.980412:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832711552/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764305.980428:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764305.980431:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764305.980434:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787764305.980435:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764305.980436:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764305.980438:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764305.980441:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22020
00000400:00000200:2.0:1787764305.980444:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764305.980447:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764305.980449:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764305.980451:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764305.980454:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764305.980454:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1787764305.980458:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764305.980461:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764305.980461:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:2.0:1787764305.980466:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764305.980467:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00000001:3.0:1787764305.980467:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:2.0:1787764305.980470:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764305.980473:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:2.0:1787764305.980474:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:2.0:1787764305.980476:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:3.0:1787764305.980480:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832711552/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787764305.980481:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764305.980485:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764305.980491:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764305.980495:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764305.980497:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:2.0:1787764305.980498:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00000001:3.0:1787764305.980500:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764305.980502:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:3.0:1787764305.980505:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:2.0:1787764305.980506:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764305.980510:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764305.980511:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764305.980514:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764305.980515:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764305.980520:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764305.980524:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00100000:3.0:1787764305.980524:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832711552:192.168.203.110@tcp:250:kworker.0
00000008:00100000:2.0:1787764305.980531:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787764305.980531:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00000001:2.0:1787764305.980536:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764305.980538:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764305.980538:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764305.980541:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764305.980545:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
02000000:00000001:3.0:1787764305.980545:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:2.0:1787764305.980550:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000010:3.0:1787764305.980551:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:2.0:1787764305.980554:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764305.980555:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764305.980558:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764305.980562:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764305.980563:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764305.980567:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:3.0:1787764305.980567:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
02000000:00000001:2.0:1787764305.980573:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764305.980576:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
02000000:00000010:2.0:1787764305.980580:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f7c00.
00000100:00000200:3.0:1787764305.980583:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832711552, portal 25
00000100:00000001:2.0:1787764305.980586:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764305.980587:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764305.980590:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00100000:2.0:1787764305.980599:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832711680/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000040:3.0:1787764305.980600:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832711552/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.980621:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1787764305.980626:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:3.0:1787764305.980627:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787764305.980629:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764305.980632:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000040:2.0:1787764305.980636:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832711680/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764305.980637:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832711552, offset 0
00000400:00000200:3.0:1787764305.980647:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787764305.980648:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764305.980652:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764305.980656:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:3.0:1787764305.980656:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764305.980658:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764305.980661:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764305.980663:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764305.980666:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22020
00000400:00000200:3.0:1787764305.980666:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764305.980675:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764305.980680:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764305.980683:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764305.980685:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764305.980687:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58794:58794:256:4294967295] 192.168.203.110@tcp LPNI seq info [58794:58794:8:4294967295]
00000400:00000200:3.0:1787764305.980690:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764305.980696:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000400:00000200:2.0:1787764305.980703:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:3.0:1787764305.980704:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764305.980710:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000800:00000010:3.0:1787764305.980711:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000100:00080000:2.0:1787764305.980712:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764305.980716:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764305.980720:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1787764305.980721:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:2.0:1787764305.980725:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000200:3.0:1787764305.980727:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000100:00000001:2.0:1787764305.980729:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764305.980733:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:3.1:1787764305.980736:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764305.980739:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000020:00000040:2.0:1787764305.980741:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787764305.980746:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000100:3.0:1787764305.980746:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000020:00000040:2.0:1787764305.980750:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000010:3.0:1787764305.980750:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000020:00000001:2.0:1787764305.980754:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:2.0:1787764305.980757:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:3.0:1787764305.980758:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000040:2.0:1787764305.980762:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764305.980765:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787764305.980767:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000040:2.0:1787764305.980770:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764305.980774:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:3.0:1787764305.980774:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000020:00000040:2.0:1787764305.980777:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000010:3.0:1787764305.980778:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00080000:2.0:1787764305.980782:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764305.980785:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764305.980785:0:257575:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:2.0:1787764305.980788:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764305.980792:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000200:3.0:1787764305.980792:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832711552/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764305.980798:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764305.980802:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764305.980805:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787764305.980808:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764305.980811:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764305.980811:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764305.980815:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832711552/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764330 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1787764305.980821:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000100:00000001:3.0:1787764305.980824:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1787764305.980825:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000001:3.0:1787764305.980830:0:257575:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:2.0:1787764305.980831:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764305.980833:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764305.980835:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1787764305.980836:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764305.980837:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764305.980840:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764305.980844:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000010:2.0:1787764305.980846:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f6400.
00000100:00000001:3.0:1787764305.980847:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316694259
00000100:00000001:2.0:1787764305.980851:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764305.980854:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d404e0.
00000100:00100000:2.0:1787764305.980859:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832711808/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764305.980863:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:2.0:1787764305.980872:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764305.980872:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316694259
00000100:00000001:2.0:1787764305.980877:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764305.980878:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40960.
00000100:00000001:3.0:1787764305.980882:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787764305.980884:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1787764305.980887:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832711808/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764305.980894:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764305.980895:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764305.980897:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764305.980899:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000040:2.0:1787764305.980901:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:2.0:1787764305.980904:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00100000:3.0:1787764305.980906:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764305/real 1787764305]  req@ffff9751f8658000 x1874599832711552/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787764305.980907:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787764305.980910:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764305.980913:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764305.980918:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764305.980922:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22020
00000400:00000001:3.0:1787764305.980923:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000200:2.0:1787764305.980927:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000001:3.0:1787764305.980927:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764305.980930:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:2.0:1787764305.980931:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764305.980935:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000200:3.0:1787764305.980935:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764305.980940:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000040:2.0:1787764305.980941:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764305.980944:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764305.980944:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000040:2.0:1787764305.980949:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764305.980952:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000200:3.0:1787764305.980953:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832711552/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787764305.980954:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764305.980957:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787764305.980959:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764305.980961:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787764305.980965:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:2.0:1787764305.980969:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:2.0:1787764305.980973:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764305.980977:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:3.0:1787764305.980978:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832711552/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787764305.980980:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764305.980985:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764305.980989:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:2.0:1787764305.980994:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764305.980999:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764305.981002:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764305.981003:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
02000000:00000001:2.0:1787764305.981004:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764305.981008:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764305.981009:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764305.981010:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764305.981014:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1787764305.981015:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000008:00000001:2.0:1787764305.981017:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764305.981020:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764305.981020:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:2.0:1787764305.981026:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000020:00000040:2.0:1787764305.981029:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00100000:3.0:1787764305.981031:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832711552/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764305.981032:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764305.981034:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764305.981038:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764305.981040:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764305.981042:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764305.981046:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f6c00.
00000100:00000001:3.0:1787764305.981048:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764305.981049:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764305.981055:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832711552/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764305.981058:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832711936/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764305.981076:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764305.981079:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764305.981082:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764305.981084:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764305.981087:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764305.981092:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764305.981093:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832711936/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.981097:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764305.981100:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764305.981107:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:2.0:1787764305.981123:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764305.981127:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000040:2.0:1787764305.981128:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (22030)
00000100:00000001:3.0:1787764305.981170:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764305.981179:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832711552/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764305.981188:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832711552:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764305.981193:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764305.981195:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764305.981200:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832711552/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.981214:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764305.981216:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764305.981219:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764305.981223:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764305.981225:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764305.981226:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764305.981229:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764305.981232:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f4800.
02000000:00000001:3.0:1787764305.981236:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764305.981238:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764305.981240:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:3.0:1787764305.981243:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764305.981245:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764305.981247:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764305.981253:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764305.981255:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764305.981258:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764305.981259:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764305.981275:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764305.981280:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832711680/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.981289:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764305.981290:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764305.981293:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764305.981296:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764305.981299:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764305.981304:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832711680:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764305.981308:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764305.981310:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764305.981313:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764305.981316:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764305.981320:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764305.981323:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764305.981328:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764305.981333:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764305.981337:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764305.981340:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832711680, portal 10
00000100:00000001:3.0:1787764305.981342:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764305.981344:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:3.0:1787764305.981351:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832711680/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.981359:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764305.981362:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764305.981365:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764305.981369:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832711680, offset 0
00000400:00000200:3.0:1787764305.981448:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764305.981554:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764305.981559:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764305.981562:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764305.981564:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764305.981566:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764305.981570:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764305.981574:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58795:58795:256:4294967295] 192.168.203.110@tcp LPNI seq info [58795:58795:8:4294967295]
00000400:00000200:3.0:1787764305.981577:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764305.981616:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764305.981620:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764305.981624:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764305.981628:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764305.981631:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787764305.981637:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764305.981639:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1787764305.981645:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764305.981647:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764305.981650:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764305.981654:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764305.981656:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764305.981658:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764305.981662:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764305.981666:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832711680/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.981672:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764305.981674:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832711680/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764330 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.981679:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764305.981680:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764305.981682:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764305.981685:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764305.981688:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764305.981689:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316693024
00000100:00000001:3.0:1787764305.981691:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316694259
00000100:00000001:3.0:1787764305.981693:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316693024
00000100:00000001:3.0:1787764305.981694:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316694259
00000100:00000001:3.0:1787764305.981696:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764305.981697:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764305.981698:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764305.981701:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764305/real 1787764305]  req@ffff9751f8659a40 x1874599832711680/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764305.981706:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764305.981707:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764305.981708:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764305.981709:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764305.981711:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764305.981712:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764305.981714:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832711680/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764305.981719:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832711680/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.981728:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764305.981729:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764305.981730:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764305.981731:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764305.981733:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832711680/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.981737:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764305.981740:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832711680/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.981744:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764305.981745:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764305.981746:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764305.981747:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764305.981748:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764305.981758:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764305.981760:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764305.981762:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764305.981763:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764305.981765:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832711680/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764305.981770:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832711680:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764305.981772:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764305.981775:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832711808/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.981779:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764305.981780:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764305.981782:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764305.981783:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764305.981785:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764305.981786:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832711808:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764305.981789:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764305.981790:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764305.981791:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764305.981792:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764305.981794:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764305.981796:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764305.981798:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764305.981800:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764305.981802:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764305.981804:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832711808, portal 10
00000100:00000001:3.0:1787764305.981805:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764305.981807:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:3.0:1787764305.981811:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832711808/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.981815:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764305.981817:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764305.981818:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764305.981820:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832711808, offset 0
00000400:00000200:3.0:1787764305.981822:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764305.981824:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764305.981827:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764305.981829:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764305.981830:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764305.981832:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764305.981833:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764305.981835:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58796:58796:256:4294967295] 192.168.203.110@tcp LPNI seq info [58796:58796:8:4294967295]
00000400:00000200:3.0:1787764305.981838:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764305.981843:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764305.981846:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764305.981848:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764305.981850:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764305.981852:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787764305.981855:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764305.981857:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1787764305.981859:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764305.981860:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764305.981862:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764305.981865:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764305.981867:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764305.981868:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764305.981869:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764305.981871:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832711808/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.981876:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764305.981878:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832711808/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764330 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.981882:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764305.981883:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764305.981885:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764305.981887:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764305.981888:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764305.981890:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316693012
00000100:00000001:3.0:1787764305.981891:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316694259
00000100:00000001:3.0:1787764305.981892:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316693012
00000100:00000001:3.0:1787764305.981894:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316694259
00000100:00000001:3.0:1787764305.981895:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764305.981897:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764305.981898:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764305.981900:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764305/real 1787764305]  req@ffff9751f86583c0 x1874599832711808/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764305.981905:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764305.981906:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764305.981907:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764305.981909:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764305.981910:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764305.981911:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764305.981912:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832711808/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764305.981917:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832711808/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.981921:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764305.981922:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764305.981923:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764305.981924:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764305.981926:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832711808/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.981930:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764305.981933:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832711808/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.981937:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764305.981938:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764305.981939:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764305.981940:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764305.981941:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764305.981942:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764305.981943:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764305.981945:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764305.981946:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764305.981949:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832711808/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764305.981954:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832711808:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764305.981957:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764305.981959:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832711936/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.981963:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764305.981963:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764305.981965:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764305.981966:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764305.981967:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764305.981969:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832711936:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764305.981970:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764305.981971:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764305.981972:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764305.981973:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764305.981975:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764305.981976:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764305.981978:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764305.981980:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764305.981982:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764305.981983:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832711936, portal 4
00000100:00000001:3.0:1787764305.981985:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764305.981986:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:3.0:1787764305.981989:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832711936/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.981993:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764305.981995:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764305.981996:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764305.981997:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832711936, offset 0
00000400:00000200:3.0:1787764305.982000:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764305.982002:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764305.982004:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764305.982005:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764305.982007:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764305.982009:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764305.982011:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764305.982013:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58797:58797:256:4294967295] 192.168.203.110@tcp LPNI seq info [58797:58797:8:4294967295]
00000400:00000200:3.0:1787764305.982015:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764305.982019:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764305.982022:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764305.982023:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764305.982025:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764305.982027:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000200:3.1:1787764305.982030:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764305.982031:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 48 secs later)
00000800:00000100:3.0:1787764305.982034:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764305.982035:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764305.982037:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764305.982039:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764305.982040:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764305.982041:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764305.982043:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764305.982045:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832711936/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.982050:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764305.982052:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832711936/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764330 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.982056:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764305.982057:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764305.982059:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764305.982060:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764305.982062:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764305.982063:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316693029
00000100:00000001:3.0:1787764305.982065:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316694259
00000100:00000001:3.0:1787764305.982066:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316693029
00000100:00000001:3.0:1787764305.982068:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316694259
00000100:00000001:3.0:1787764305.982069:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764305.982070:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764305.982071:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764305.982073:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764305/real 1787764305]  req@ffff9751f865ad00 x1874599832711936/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764305.982078:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764305.982078:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764305.982079:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764305.982080:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764305.982081:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764305.982083:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764305.982084:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832711936/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764305.982089:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832711936/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.982093:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764305.982094:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764305.982096:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764305.982096:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764305.982099:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832711936/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.982103:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764305.982105:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832711936/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.982109:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764305.982127:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764305.982129:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764305.982130:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764305.982132:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764305.982148:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764305.982151:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764305.982155:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764305.982157:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764305.982160:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832711936/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764305.982166:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832711936:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764305.982170:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764305.982171:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764305.982174:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832711680/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.982179:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764305.982181:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764305.982182:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764305.982184:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764305.982185:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764305.982186:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764305.982188:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764305.982189:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f7c00.
02000000:00000001:3.0:1787764305.982191:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764305.982192:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764305.982193:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:3.0:1787764305.982195:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764305.982195:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764305.982196:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764305.982198:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832711808/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.982207:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764305.982208:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764305.982210:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764305.982212:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764305.982213:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764305.982214:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764305.982216:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764305.982218:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f6400.
02000000:00000001:3.0:1787764305.982220:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764305.982221:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764305.982223:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:3.0:1787764305.982225:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764305.982226:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764305.982228:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764305.982232:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832711936/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764330 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764305.982238:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764305.982239:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764305.982241:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764305.982243:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764305.982244:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764305.982246:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764305.982247:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764305.982249:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f6c00.
02000000:00000001:3.0:1787764305.982251:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764305.982252:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764305.982254:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:3.0:1787764305.982256:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764305.982258:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764305.982260:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764305.982264:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764305.982265:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764305.982267:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764305.982269:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764306.620240:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764306.620243:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764306.620248:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764306.620248:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764306.620251:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764306.620255:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764306.620256:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764306.620257:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764306.620262:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764306.620264:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764306.620266:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764306.620267:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764306.620272:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764306.620275:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764306.620286:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764306.620364:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764306.620369:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764306.620372:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764307.004217:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764307.004224:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764307.004226:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764307.004231:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764307.004234:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764307.004237:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764307.004239:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764307.004241:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764307.004243:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764307.260221:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316694259, at 4316694272
00000100:00000010:3.0:1787764307.260240:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40960.
00000100:00000040:2.0:1787764307.260252:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316694259, at 4316694272
00000100:00000010:2.0:1787764307.260260:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d404e0.
00000100:00000001:2.0:1787764307.644242:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764307.644244:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764307.644249:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764307.644250:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764307.644253:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764307.644255:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764307.644257:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764307.644259:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764307.644261:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764307.644263:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764307.644265:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764307.644267:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764307.644270:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764307.644271:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764307.644274:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764307.644275:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764307.644280:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764307.644281:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764307.644282:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764307.644287:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764307.644288:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764307.644333:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764307.644338:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764307.644342:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764308.028238:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764308.028245:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764308.028247:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764308.028251:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764308.028253:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764308.028257:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764308.028259:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764308.028261:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764308.028264:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764308.668228:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764308.668232:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764308.668236:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764308.668238:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764308.668244:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764308.668245:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764308.668249:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764308.668251:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764308.668257:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764308.668258:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764308.668260:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764308.668262:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764308.668264:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764308.668267:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764308.668270:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764308.668323:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764308.668328:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764308.668331:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764309.052225:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764309.052233:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764309.052236:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764309.052241:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764309.052246:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764309.052249:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764309.052252:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764309.052254:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764309.052257:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764309.692214:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764309.692222:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764309.692230:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764309.692232:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764309.692234:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764309.692237:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764309.692241:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764309.692242:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764309.692247:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764309.692252:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764309.692257:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764309.692261:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764309.692264:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764309.692267:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764309.692271:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764309.692295:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764309.692300:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764309.692305:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764310.076235:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764310.076242:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764310.076243:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764310.076248:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764310.076253:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764310.076256:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764310.076258:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764310.076260:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764310.076262:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764310.716228:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764310.716234:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764310.716236:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764310.716240:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764310.716241:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764310.716244:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764310.716246:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764310.716248:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764310.716252:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764310.716254:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764310.716257:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764310.716259:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764310.716261:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764310.716263:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764310.716265:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764310.716325:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764310.716329:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764310.716331:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787764311.100211:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764311.100219:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787764311.100222:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764311.100225:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764311.100229:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764311.100233:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:2.0:1787764311.100235:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 22025
00000100:00000001:3.0:1787764311.100241:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000400:00000200:2.0:1787764311.100246:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764311.100248:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764311.100253:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00100000:3.0:1787764311.100255:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:2.0:1787764311.100257:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764311.100260:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764311.100262:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764311.100264:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764311.100266:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764311.100270:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1787764311.100271:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000040:2.0:1787764311.100272:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787764311.100275:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787764311.100278:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:2.0:1787764311.100279:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787764311.100283:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787764311.100286:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000001:3.0:1787764311.100286:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:2.0:1787764311.100291:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764311.100292:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764311.100293:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764311.100297:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764311.100299:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764311.100300:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1787764311.100302:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787764311.100307:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764311.100309:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764311.100311:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764311.100313:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764311.100315:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764311.100317:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764311.100319:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764311.100326:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658780.
00000020:00000040:2.0:1787764311.100328:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787764311.100331:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764311.100333:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764311.100335:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764311.100338:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764311.100341:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764311.100348:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f6800.
00000100:00000001:2.0:1787764311.100353:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764311.100361:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658780 x1874599832712064/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764311.100368:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764311.100371:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764311.100375:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000744875b5] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658780 x1874599832712064/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764311.100392:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764311.100394:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764311.100396:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787764311.100398:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764311.100399:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764311.100401:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764311.100402:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22025
00000400:00000200:2.0:1787764311.100405:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764311.100407:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764311.100408:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764311.100411:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764311.100413:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764311.100415:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764311.100416:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764311.100418:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764311.100419:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:2.0:1787764311.100420:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764311.100421:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:2.0:1787764311.100422:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:2.0:1787764311.100424:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:2.0:1787764311.100426:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764311.100427:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764311.100429:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764311.100430:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764311.100432:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:2.0:1787764311.100434:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764311.100435:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764311.100436:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764311.100440:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764311.100442:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
02000000:00000001:2.0:1787764311.100444:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764311.100446:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
02000000:00000001:2.0:1787764311.100447:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764311.100451:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764311.100455:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787764311.100458:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764311.100459:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832712064/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764311.100460:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764311.100466:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000020:00000040:2.0:1787764311.100471:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764311.100475:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764311.100477:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764311.100481:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764311.100482:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1787764311.100482:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:2.0:1787764311.100486:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764311.100487:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764311.100490:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f5000.
00000100:00000001:2.0:1787764311.100494:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764311.100494:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764311.100500:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00100000:2.0:1787764311.100502:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832712192/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1787764311.100508:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764311.100511:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764311.100513:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764311.100516:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832712192/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764311.100525:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764311.100528:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764311.100530:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00100000:3.0:1787764311.100530:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832712064:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764311.100533:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764311.100535:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764311.100540:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764311.100543:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00080000:2.0:1787764311.100544:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22025
00000400:00000200:2.0:1787764311.100549:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:3.0:1787764311.100551:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000400:00000200:2.0:1787764311.100552:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764311.100556:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
02000000:00000001:3.0:1787764311.100557:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764311.100559:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764311.100563:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1787764311.100565:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:2.0:1787764311.100566:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764311.100570:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764311.100571:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764311.100572:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787764311.100573:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000010:3.0:1787764311.100575:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
00000020:00000040:2.0:1787764311.100576:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:2.0:1787764311.100579:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:2.0:1787764311.100583:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
02000000:00000001:3.0:1787764311.100583:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764311.100590:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764311.100592:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764311.100595:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764311.100598:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:3.0:1787764311.100598:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:2.0:1787764311.100602:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:2.0:1787764311.100605:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764311.100607:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764311.100608:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:3.0:1787764311.100608:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
02000000:00000001:2.0:1787764311.100611:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764311.100614:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764311.100615:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764311.100617:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787764311.100618:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764311.100619:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000001:2.0:1787764311.100620:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764311.100626:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000020:00000040:2.0:1787764311.100628:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764311.100629:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764311.100630:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764311.100631:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764311.100631:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764311.100632:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764311.100636:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f4000.
00000100:00000001:2.0:1787764311.100639:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764311.100641:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832712064, portal 25
00000100:00100000:2.0:1787764311.100642:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832712320/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764311.100648:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:2.0:1787764311.100651:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764311.100653:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000001:2.0:1787764311.100654:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764311.100660:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832712320/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764311.100668:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764311.100669:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832712064/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764336 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787764311.100671:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1787764311.100677:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:2.0:1787764311.100681:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764311.100688:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787764311.100691:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764311.100693:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764311.100696:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1787764311.100697:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764311.100703:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22025
00000100:00000040:3.0:1787764311.100708:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764311.100709:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764311.100714:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764311.100715:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000010:3.0:1787764311.100717:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000040:2.0:1787764311.100718:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764311.100724:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764311.100726:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000200:3.0:1787764311.100727:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832712064, offset 0
00000100:00000001:2.0:1787764311.100728:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764311.100732:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764311.100734:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787764311.100735:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764311.100737:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787764311.100738:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:2.0:1787764311.100742:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000200:3.0:1787764311.100743:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000040:2.0:1787764311.100746:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764311.100749:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764311.100750:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764311.100752:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764311.100753:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:2.0:1787764311.100756:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:3.0:1787764311.100758:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764311.100759:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764311.100762:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764311.100766:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764311.100767:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764311.100769:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764311.100771:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000400:00000200:3.0:1787764311.100773:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000008:00000001:2.0:1787764311.100774:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764311.100777:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764311.100781:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000020:00000040:2.0:1787764311.100784:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000400:00000200:3.0:1787764311.100785:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
02000000:00000001:2.0:1787764311.100786:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764311.100788:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764311.100790:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764311.100791:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764311.100792:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764311.100795:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
02000000:00000010:2.0:1787764311.100796:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f4c00.
00000100:00000001:2.0:1787764311.100800:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764311.100804:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00100000:2.0:1787764311.100805:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832712448/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764311.100814:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1787764311.100814:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:2.0:1787764311.100816:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764311.100822:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832712448/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764311.100828:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58798:58798:256:4294967295] 192.168.203.110@tcp LPNI seq info [58798:58798:8:4294967295]
00000100:00000001:2.0:1787764311.100832:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764311.100837:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (22035)
00000400:00000200:3.0:1787764311.100842:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764311.100856:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764311.100864:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764311.100870:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764311.100877:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764311.100882:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787764311.100887:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764311.100891:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1787764311.100896:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764311.100899:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764311.100906:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764311.100911:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764311.100917:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764311.100919:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764311.100925:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764311.100930:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832712064/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764336 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.100941:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764311.100945:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832712064/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764336 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.100953:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764311.100956:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764311.100961:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764311.100966:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764311.100969:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316699379
00000100:00000010:3.0:1787764311.100975:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40960.
00000100:00000001:3.0:1787764311.100982:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787764311.100990:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316699379
00000100:00000010:3.0:1787764311.100993:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40a80.
00000100:00000001:3.0:1787764311.100996:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787764311.100999:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764311.101002:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764311.101005:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764311.101011:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764311/real 1787764311]  req@ffff9751f8658780 x1874599832712064/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764311.101022:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764311.101024:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764311.101026:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764311.101029:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764311.101031:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764311.101034:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764311.101038:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832712064/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764311.101048:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832712064/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.101056:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764311.101058:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764311.101061:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764311.101063:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764311.101068:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832712064/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.101091:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764311.101097:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832712064/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.101112:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764311.101115:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764311.101117:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764311.101119:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764311.101121:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764311.101123:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764311.101127:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764311.101131:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764311.101158:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764311.101163:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832712064/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764311.101173:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832712064:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764311.101178:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764311.101181:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764311.101185:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832712064/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.101192:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764311.101195:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764311.101198:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764311.101201:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764311.101203:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764311.101205:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764311.101207:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764311.101210:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f6800.
02000000:00000001:3.0:1787764311.101214:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764311.101216:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764311.101217:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:3.0:1787764311.101220:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764311.101222:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764311.101224:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764311.101232:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764311.101233:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764311.101236:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764311.101237:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764311.101239:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764311.101243:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832712192/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.101266:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764311.101268:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764311.101270:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764311.101272:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764311.101275:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764311.101278:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832712192:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764311.101282:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764311.101285:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764311.101287:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764311.101289:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764311.101292:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764311.101295:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764311.101299:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764311.101302:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764311.101306:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764311.101309:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832712192, portal 10
00000100:00000001:3.0:1787764311.101312:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764311.101313:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:3.0:1787764311.101320:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832712192/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764336 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.101327:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764311.101330:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764311.101333:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764311.101336:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832712192, offset 0
00000400:00000200:3.0:1787764311.101340:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764311.101345:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764311.101350:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764311.101354:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764311.101357:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764311.101359:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764311.101362:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764311.101366:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58799:58799:256:4294967295] 192.168.203.110@tcp LPNI seq info [58799:58799:8:4294967295]
00000400:00000200:3.0:1787764311.101370:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764311.101377:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764311.101382:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764311.101386:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764311.101390:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764311.101393:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787764311.101398:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764311.101401:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1787764311.101406:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764311.101408:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764311.101411:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764311.101415:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764311.101418:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764311.101420:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764311.101423:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764311.101427:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832712192/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764336 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.101436:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764311.101440:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832712192/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764336 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.101447:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764311.101449:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764311.101453:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764311.101457:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764311.101461:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764311.101463:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316698144
00000100:00000001:3.0:1787764311.101466:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316699379
00000100:00000001:3.0:1787764311.101469:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316698144
00000100:00000001:3.0:1787764311.101471:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316699379
00000100:00000001:3.0:1787764311.101473:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764311.101475:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764311.101478:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764311.101482:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764311/real 1787764311]  req@ffff9751f865ad00 x1874599832712192/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764311.101491:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764311.101492:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764311.101494:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764311.101497:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764311.101499:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764311.101501:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764311.101505:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832712192/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764311.101514:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832712192/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.101521:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764311.101523:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764311.101524:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764311.101526:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764311.101530:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832712192/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.101537:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764311.101541:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832712192/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.101549:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764311.101551:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764311.101554:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764311.101555:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764311.101557:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764311.101570:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764311.101573:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764311.101576:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764311.101578:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764311.101583:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832712192/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764311.101591:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832712192:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764311.101595:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764311.101599:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832712320/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.101606:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764311.101608:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764311.101610:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764311.101612:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764311.101614:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764311.101618:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832712320:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764311.101622:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764311.101624:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764311.101626:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764311.101628:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764311.101632:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764311.101634:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764311.101638:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764311.101641:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764311.101645:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764311.101648:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832712320, portal 10
00000100:00000001:3.0:1787764311.101650:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764311.101652:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:3.0:1787764311.101658:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832712320/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764336 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.101665:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764311.101668:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764311.101671:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764311.101673:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832712320, offset 0
00000400:00000200:3.0:1787764311.101677:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764311.101682:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764311.101686:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764311.101689:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764311.101691:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764311.101694:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764311.101696:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764311.101699:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58800:58800:256:4294967295] 192.168.203.110@tcp LPNI seq info [58800:58800:8:4294967295]
00000400:00000200:3.0:1787764311.101704:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764311.101711:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764311.101715:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764311.101719:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764311.101723:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764311.101726:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787764311.101730:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764311.101733:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1787764311.101737:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764311.101739:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764311.101742:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764311.101746:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764311.101750:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764311.101752:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764311.101755:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764311.101759:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832712320/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764336 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.101767:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764311.101771:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832712320/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764336 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.101778:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764311.101780:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764311.101783:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764311.101786:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764311.101789:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764311.101791:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316698132
00000100:00000001:3.0:1787764311.101794:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316699379
00000100:00000001:3.0:1787764311.101796:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316698132
00000100:00000001:3.0:1787764311.101799:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316699379
00000100:00000001:3.0:1787764311.101801:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764311.101803:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764311.101805:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764311.101809:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764311/real 1787764311]  req@ffff9751f86583c0 x1874599832712320/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764311.101817:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764311.101819:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764311.101821:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764311.101823:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764311.101825:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764311.101827:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764311.101831:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832712320/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764311.101840:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832712320/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.101847:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764311.101849:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764311.101851:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764311.101852:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764311.101857:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832712320/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.101864:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764311.101868:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832712320/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.101876:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764311.101878:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764311.101880:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764311.101881:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764311.101884:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764311.101885:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764311.101888:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764311.101892:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764311.101894:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764311.101898:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832712320/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764311.101907:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832712320:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764311.101912:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764311.101915:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832712448/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.101923:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764311.101924:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764311.101926:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764311.101928:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764311.101930:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764311.101934:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832712448:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764311.101937:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764311.101939:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764311.101941:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764311.101944:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764311.101947:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764311.101949:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764311.101953:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764311.101956:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764311.101959:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764311.101962:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832712448, portal 4
00000100:00000001:3.0:1787764311.101964:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764311.101966:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:3.0:1787764311.101972:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832712448/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764336 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.101979:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764311.101982:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764311.101984:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764311.101987:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832712448, offset 0
00000400:00000200:3.0:1787764311.101991:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764311.101995:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764311.101999:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764311.102002:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764311.102005:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764311.102007:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764311.102010:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764311.102013:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58801:58801:256:4294967295] 192.168.203.110@tcp LPNI seq info [58801:58801:8:4294967295]
00000400:00000200:3.0:1787764311.102018:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764311.102024:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764311.102029:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764311.102032:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764311.102036:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764311.102039:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000200:3.1:1787764311.102044:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764311.102047:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 43 secs later)
00000800:00000100:3.0:1787764311.102051:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764311.102054:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764311.102057:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764311.102061:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764311.102064:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764311.102066:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764311.102069:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764311.102073:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832712448/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764336 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.102098:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764311.102102:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832712448/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764336 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.102122:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764311.102124:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764311.102128:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764311.102132:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764311.102153:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764311.102155:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316698150
00000100:00000001:3.0:1787764311.102158:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316699379
00000100:00000001:3.0:1787764311.102160:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316698150
00000100:00000001:3.0:1787764311.102162:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316699379
00000100:00000001:3.0:1787764311.102165:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764311.102167:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764311.102169:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764311.102174:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764311/real 1787764311]  req@ffff9751f8659a40 x1874599832712448/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764311.102183:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764311.102185:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764311.102187:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764311.102189:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764311.102191:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764311.102194:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764311.102198:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832712448/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764311.102207:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832712448/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.102214:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764311.102215:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764311.102217:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764311.102219:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764311.102223:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832712448/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.102230:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764311.102234:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832712448/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.102242:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764311.102245:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764311.102247:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764311.102248:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764311.102250:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764311.102252:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764311.102254:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764311.102258:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764311.102260:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764311.102265:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832712448/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764311.102274:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832712448:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764311.102279:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764311.102281:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764311.102285:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832712192/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.102293:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764311.102294:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764311.102297:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764311.102300:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764311.102302:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764311.102303:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764311.102305:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764311.102307:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f5000.
02000000:00000001:3.0:1787764311.102310:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764311.102312:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764311.102313:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:3.0:1787764311.102317:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764311.102318:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764311.102320:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764311.102324:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832712320/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.102331:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764311.102332:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764311.102334:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764311.102336:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764311.102337:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764311.102339:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764311.102341:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764311.102342:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f4000.
02000000:00000001:3.0:1787764311.102345:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764311.102346:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764311.102347:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:3.0:1787764311.102350:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764311.102351:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764311.102353:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764311.102357:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832712448/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764336 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764311.102364:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764311.102366:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764311.102367:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764311.102370:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764311.102371:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764311.102372:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764311.102374:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764311.102376:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f4c00.
02000000:00000001:3.0:1787764311.102378:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764311.102379:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764311.102381:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:3.0:1787764311.102383:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764311.102385:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764311.102387:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764311.102390:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764311.102392:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764311.102394:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764311.102396:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764311.740235:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764311.740240:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764311.740242:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764311.740246:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764311.740247:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764311.740251:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764311.740256:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764311.740256:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764311.740261:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764311.740261:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764311.740265:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764311.740268:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764311.740269:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764311.740271:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764311.740273:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764311.740326:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764311.740330:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764311.740333:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764312.124237:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764312.124244:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764312.124246:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764312.124250:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764312.124253:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764312.124256:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764312.124258:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764312.124260:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764312.124263:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764312.380264:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316699379, at 4316699392
00000100:00000010:3.0:1787764312.380283:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40a80.
00000100:00000040:2.0:1787764312.380331:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316699379, at 4316699392
00000100:00000010:2.0:1787764312.380339:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40960.
00010000:00000001:3.0:1787764312.764263:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764312.764269:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764312.764274:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764312.764275:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764312.764278:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764312.764282:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764312.764283:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764312.764286:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764312.764288:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764312.764294:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764312.764316:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764312.764321:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764312.764326:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764312.764329:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764312.764335:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764312.764347:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764312.764351:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764312.764355:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764313.148270:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764313.148281:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764313.148285:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764313.148294:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764313.148300:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764313.148314:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764313.148318:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764313.148322:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764313.148329:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764313.788240:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764313.788247:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764313.788248:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764313.788253:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764313.788257:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764313.788260:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764313.788263:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764313.788265:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764313.788267:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764313.788298:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764313.788314:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764313.788322:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764313.788324:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764313.788328:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764313.788333:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764313.788336:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764313.788341:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764313.788345:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764313.788347:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764313.788350:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764313.788353:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764313.788397:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764313.788401:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764313.788403:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764314.172228:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764314.172234:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764314.172236:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764314.172241:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764314.172245:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764314.172248:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764314.172250:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764314.172253:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764314.172255:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764314.812229:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764314.812235:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764314.812238:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764314.812240:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764314.812242:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764314.812245:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764314.812248:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764314.812249:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764314.812251:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764314.812254:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764314.812255:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764314.812258:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764314.812261:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764314.812263:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764314.812266:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764314.812319:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764314.812322:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764314.812324:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764315.196244:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764315.196253:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764315.196255:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764315.196260:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764315.196264:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764315.196268:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764315.196271:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764315.196274:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764315.196277:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764315.836226:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764315.836228:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764315.836235:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764315.836236:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764315.836239:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764315.836241:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764315.836247:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764315.836248:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764315.836253:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764315.836254:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764315.836257:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764315.836259:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764315.836260:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764315.836264:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764315.836265:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764315.836267:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764315.836270:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764315.836278:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764315.836280:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764315.836283:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764315.836285:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764315.836321:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764315.836324:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764315.836327:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787764316.220225:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764316.220225:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764316.220231:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764316.220234:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764316.220236:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00100000:3.0:1787764316.220238:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764316.220240:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1787764316.220242:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00080000:2.0:1787764316.220246:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764316.220246:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.220250:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764316.220253:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764316.220254:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.220257:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764316.220264:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 22030
00000400:00000200:2.0:1787764316.220274:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764316.220281:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764316.220284:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764316.220286:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764316.220289:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764316.220291:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764316.220292:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764316.220295:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764316.220297:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787764316.220299:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764316.220302:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787764316.220304:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787764316.220305:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787764316.220307:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764316.220309:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764316.220310:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764316.220312:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764316.220314:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787764316.220316:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764316.220318:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764316.220321:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764316.220324:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764316.220327:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764316.220329:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764316.220332:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764316.220340:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
00000020:00000040:2.0:1787764316.220345:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787764316.220349:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764316.220350:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764316.220354:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764316.220356:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764316.220361:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764316.220369:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f4000.
00000100:00000001:2.0:1787764316.220375:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764316.220382:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832712576/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764316.220391:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764316.220395:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764316.220398:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832712576/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764316.220415:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764316.220417:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764316.220419:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787764316.220421:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764316.220422:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764316.220424:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764316.220426:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22030
00000400:00000200:2.0:1787764316.220430:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764316.220432:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764316.220434:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764316.220437:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764316.220438:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:2.0:1787764316.220439:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764316.220442:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1787764316.220443:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764316.220445:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1787764316.220446:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764316.220449:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764316.220452:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00000040:3.0:1787764316.220453:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832712576/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787764316.220454:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764316.220458:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:2.0:1787764316.220461:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:2.0:1787764316.220466:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:3.0:1787764316.220469:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:2.0:1787764316.220472:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764316.220473:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764316.220478:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1787764316.220478:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000040:2.0:1787764316.220482:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:3.0:1787764316.220483:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764316.220486:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:3.0:1787764316.220487:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1787764316.220490:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00100000:3.0:1787764316.220495:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832712576:192.168.203.110@tcp:250:kworker.0
00000100:00080000:2.0:1787764316.220496:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764316.220501:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764316.220504:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764316.220504:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:2.0:1787764316.220508:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764316.220510:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:2.0:1787764316.220511:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764316.220515:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1787764316.220516:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:2.0:1787764316.220518:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1787764316.220521:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00000001:2.0:1787764316.220522:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764316.220525:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764316.220530:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
02000000:00000010:3.0:1787764316.220531:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
00000020:00000040:2.0:1787764316.220532:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787764316.220536:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764316.220537:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764316.220539:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764316.220541:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764316.220542:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764316.220543:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764316.220548:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000010:2.0:1787764316.220550:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f5000.
00000100:00000001:2.0:1787764316.220554:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764316.220556:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000100:00100000:2.0:1787764316.220559:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832712704/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1787764316.220565:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000001:2.0:1787764316.220566:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764316.220570:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764316.220573:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832712704/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764316.220576:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832712576, portal 25
00000100:00000001:3.0:1787764316.220581:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764316.220583:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000001:2.0:1787764316.220585:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764316.220588:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1787764316.220592:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832712576/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764341 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787764316.220593:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1787764316.220597:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764316.220599:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764316.220605:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764316.220609:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00080000:2.0:1787764316.220610:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22030
00000100:00000040:3.0:1787764316.220615:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764316.220618:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764316.220620:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000400:00000200:2.0:1787764316.220624:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000200:3.0:1787764316.220626:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832712576, offset 0
00000100:00080000:2.0:1787764316.220627:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764316.220631:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764316.220634:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787764316.220635:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000040:2.0:1787764316.220639:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764316.220641:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764316.220643:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764316.220645:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000400:00000200:3.0:1787764316.220645:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:2.0:1787764316.220648:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764316.220650:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:2.0:1787764316.220652:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:3.0:1787764316.220655:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000020:00000001:2.0:1787764316.220656:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:3.0:1787764316.220662:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000040:2.0:1787764316.220664:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764316.220668:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764316.220670:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764316.220672:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764316.220674:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:3.0:1787764316.220676:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00080000:2.0:1787764316.220683:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:3.0:1787764316.220683:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:2.0:1787764316.220690:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764316.220692:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
02000000:00000001:2.0:1787764316.220695:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1787764316.220700:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58802:58802:256:4294967295] 192.168.203.110@tcp LPNI seq info [58802:58802:8:4294967295]
02000000:00000001:2.0:1787764316.220701:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764316.220707:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764316.220711:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764316.220712:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764316.220716:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787764316.220719:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764316.220722:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764316.220727:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000400:00000200:3.0:1787764316.220728:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000020:00000040:2.0:1787764316.220730:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764316.220736:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764316.220737:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764316.220740:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000200:3.0:1787764316.220741:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
02000000:00000001:2.0:1787764316.220742:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764316.220744:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764316.220751:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f6800.
00000800:00000010:3.0:1787764316.220752:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000100:00000001:2.0:1787764316.220754:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764316.220761:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832712832/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.0:1787764316.220765:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787764316.220773:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000800:00000200:3.0:1787764316.220773:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000100:00000001:2.0:1787764316.220776:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787764316.220782:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:2.0:1787764316.220783:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832712832/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.1:1787764316.220787:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1787764316.220796:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787764316.220799:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1787764316.220800:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000100:00000040:2.0:1787764316.220803:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1787764316.220807:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:3.0:1787764316.220807:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00080000:2.0:1787764316.220811:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1787764316.220815:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00080000:2.0:1787764316.220816:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787764316.220819:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1787764316.220820:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000100:00080000:2.0:1787764316.220822:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000010:3.0:1787764316.220823:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:2.0:1787764316.220826:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764316.220829:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00080000:2.0:1787764316.220831:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22030
00000100:00000200:3.0:1787764316.220836:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832712576/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764341 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764316.220837:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764316.220844:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764316.220849:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787764316.220852:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764316.220853:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764316.220858:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:3.0:1787764316.220858:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832712576/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764341 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787764316.220864:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764316.220868:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764316.220872:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764316.220873:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1787764316.220877:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000040:2.0:1787764316.220883:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000100:00000001:3.0:1787764316.220885:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764316.220888:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764316.220891:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000400:00000200:3.0:1787764316.220893:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:2.0:1787764316.220894:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:2.0:1787764316.220898:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:3.0:1787764316.220900:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316704499
00000100:00000040:2.0:1787764316.220901:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764316.220905:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764316.220907:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764316.220908:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000010:3.0:1787764316.220909:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40a80.
00000020:00000040:2.0:1787764316.220910:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:2.0:1787764316.220916:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764316.220917:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764316.220919:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764316.220920:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
02000000:00000001:2.0:1787764316.220921:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764316.220925:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764316.220927:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764316.220929:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1787764316.220930:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316704499
00000008:00000001:2.0:1787764316.220932:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764316.220935:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764316.220936:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40d80.
00000100:00000010:2.0:1787764316.220939:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000100:00000001:3.0:1787764316.220941:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000020:00000040:2.0:1787764316.220944:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:3.0:1787764316.220945:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764316.220948:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1787764316.220949:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764316.220951:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764316.220953:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000001:2.0:1787764316.220954:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764316.220957:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764316.220958:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764316.220960:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764316/real 1787764316]  req@ffff9751f8658000 x1874599832712576/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1787764316.220962:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f6c00.
00000100:00000001:2.0:1787764316.220967:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764316.220976:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832712960/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000001:3.0:1787764316.220990:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:2.0:1787764316.220992:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000001:3.0:1787764316.220992:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000001:2.0:1787764316.220996:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764316.220997:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764316.221002:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000100:00000040:2.0:1787764316.221004:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832712960/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1787764316.221005:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764316.221009:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764316.221015:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832712576/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764316.221017:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764316.221023:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (22040)
00000100:00000200:3.0:1787764316.221027:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832712576/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221033:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764316.221035:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.221036:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764316.221038:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764316.221054:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832712576/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221060:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764316.221065:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832712576/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221070:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764316.221072:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.221073:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764316.221074:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.221076:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764316.221077:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764316.221079:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764316.221082:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764316.221083:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764316.221086:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832712576/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764316.221092:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832712576:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764316.221095:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.221097:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764316.221098:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832712576/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221103:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764316.221104:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764316.221106:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764316.221108:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764316.221109:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764316.221110:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764316.221111:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764316.221113:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f4000.
02000000:00000001:3.0:1787764316.221116:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764316.221117:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764316.221118:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:3.0:1787764316.221120:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764316.221121:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.221122:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.221127:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764316.221128:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764316.221130:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764316.221131:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764316.221132:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764316.221151:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832712704/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221156:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764316.221158:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764316.221160:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764316.221163:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764316.221165:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764316.221168:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832712704:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764316.221172:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764316.221174:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764316.221176:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764316.221178:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764316.221181:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764316.221183:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764316.221187:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764316.221190:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764316.221194:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764316.221197:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832712704, portal 10
00000100:00000001:3.0:1787764316.221199:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764316.221200:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:3.0:1787764316.221208:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832712704/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764341 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221215:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764316.221218:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764316.221221:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764316.221223:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832712704, offset 0
00000400:00000200:3.0:1787764316.221226:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764316.221229:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764316.221232:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764316.221235:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764316.221237:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764316.221239:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764316.221240:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764316.221242:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58803:58803:256:4294967295] 192.168.203.110@tcp LPNI seq info [58803:58803:8:4294967295]
00000400:00000200:3.0:1787764316.221245:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764316.221250:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764316.221253:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764316.221255:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764316.221258:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764316.221261:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787764316.221264:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764316.221266:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1787764316.221269:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764316.221270:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764316.221272:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764316.221274:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764316.221277:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764316.221278:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764316.221280:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764316.221283:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832712704/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764341 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221288:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764316.221290:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832712704/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764341 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221295:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764316.221296:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764316.221298:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764316.221301:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764316.221303:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764316.221304:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316703264
00000100:00000001:3.0:1787764316.221306:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316704499
00000100:00000001:3.0:1787764316.221308:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316703264
00000100:00000001:3.0:1787764316.221310:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316704499
00000100:00000001:3.0:1787764316.221311:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764316.221313:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764316.221314:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764316.221316:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764316/real 1787764316]  req@ffff9751f8659a40 x1874599832712704/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764316.221321:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764316.221322:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764316.221324:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764316.221325:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764316.221326:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764316.221328:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764316.221330:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832712704/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764316.221335:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832712704/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221339:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764316.221340:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.221341:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764316.221342:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764316.221345:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832712704/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221351:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764316.221355:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832712704/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221362:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764316.221365:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.221366:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764316.221367:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.221369:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764316.221383:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764316.221386:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764316.221389:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764316.221392:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764316.221397:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832712704/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764316.221405:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832712704:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764316.221410:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764316.221414:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832712832/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221421:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764316.221422:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764316.221425:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764316.221427:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764316.221429:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764316.221432:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832712832:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764316.221434:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764316.221436:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764316.221437:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764316.221438:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764316.221441:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764316.221443:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764316.221445:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764316.221447:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764316.221449:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764316.221451:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832712832, portal 10
00000100:00000001:3.0:1787764316.221453:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764316.221454:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:3.0:1787764316.221458:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832712832/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764341 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221463:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764316.221464:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764316.221466:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764316.221467:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832712832, offset 0
00000400:00000200:3.0:1787764316.221470:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764316.221473:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764316.221475:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764316.221477:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764316.221479:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764316.221481:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764316.221483:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764316.221485:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58804:58804:256:4294967295] 192.168.203.110@tcp LPNI seq info [58804:58804:8:4294967295]
00000400:00000200:3.0:1787764316.221487:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764316.221492:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764316.221494:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764316.221496:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764316.221499:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764316.221501:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787764316.221504:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764316.221506:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1787764316.221508:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764316.221510:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764316.221512:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764316.221514:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764316.221516:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764316.221517:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764316.221519:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764316.221521:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832712832/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764341 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221526:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764316.221528:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832712832/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764341 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221532:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764316.221533:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764316.221535:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764316.221537:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764316.221539:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764316.221541:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316703252
00000100:00000001:3.0:1787764316.221542:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316704499
00000100:00000001:3.0:1787764316.221544:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316703252
00000100:00000001:3.0:1787764316.221546:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316704499
00000100:00000001:3.0:1787764316.221548:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764316.221549:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764316.221550:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764316.221553:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764316/real 1787764316]  req@ffff9751f86583c0 x1874599832712832/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764316.221557:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764316.221558:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764316.221559:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764316.221560:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764316.221561:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764316.221563:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764316.221566:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832712832/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764316.221575:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832712832/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221581:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764316.221583:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.221585:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764316.221586:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764316.221591:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832712832/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221599:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764316.221603:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832712832/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221609:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764316.221611:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.221613:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764316.221614:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.221616:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764316.221618:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764316.221621:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764316.221625:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764316.221627:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764316.221631:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832712832/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764316.221638:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832712832:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764316.221643:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764316.221646:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832712960/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221652:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764316.221653:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764316.221655:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764316.221657:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764316.221659:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764316.221662:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832712960:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764316.221666:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764316.221668:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764316.221670:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764316.221671:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764316.221675:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764316.221677:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764316.221681:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764316.221683:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764316.221686:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764316.221689:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832712960, portal 4
00000100:00000001:3.0:1787764316.221690:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764316.221692:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:3.0:1787764316.221698:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832712960/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764341 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221704:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764316.221706:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764316.221707:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764316.221709:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832712960, offset 0
00000400:00000200:3.0:1787764316.221712:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764316.221715:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764316.221717:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764316.221719:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764316.221721:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764316.221723:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764316.221725:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764316.221727:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58805:58805:256:4294967295] 192.168.203.110@tcp LPNI seq info [58805:58805:8:4294967295]
00000400:00000200:3.0:1787764316.221730:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764316.221734:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764316.221737:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764316.221738:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764316.221741:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764316.221743:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000200:3.1:1787764316.221746:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764316.221748:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 38 secs later)
00000800:00000100:3.0:1787764316.221750:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764316.221751:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764316.221753:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764316.221756:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764316.221758:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764316.221759:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764316.221760:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764316.221763:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832712960/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764341 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221768:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764316.221770:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832712960/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764341 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221779:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764316.221780:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764316.221782:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764316.221784:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764316.221786:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764316.221787:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316703269
00000100:00000001:3.0:1787764316.221789:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316704499
00000100:00000001:3.0:1787764316.221790:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316703269
00000100:00000001:3.0:1787764316.221792:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316704499
00000100:00000001:3.0:1787764316.221793:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764316.221795:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764316.221796:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764316.221798:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764316/real 1787764316]  req@ffff9751f865ad00 x1874599832712960/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764316.221803:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764316.221804:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764316.221804:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764316.221806:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764316.221807:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764316.221808:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764316.221810:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832712960/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764316.221815:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832712960/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221819:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764316.221820:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.221822:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764316.221822:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764316.221824:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832712960/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221829:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764316.221832:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832712960/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221836:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764316.221838:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.221839:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764316.221839:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.221841:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764316.221841:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764316.221843:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764316.221845:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764316.221846:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764316.221849:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832712960/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764316.221853:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832712960:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764316.221855:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.221857:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764316.221859:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832712704/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221863:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764316.221864:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764316.221865:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764316.221867:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764316.221868:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764316.221869:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764316.221870:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764316.221871:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f5000.
02000000:00000001:3.0:1787764316.221874:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764316.221875:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764316.221876:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:3.0:1787764316.221877:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764316.221878:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.221879:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764316.221881:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832712832/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221886:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764316.221886:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764316.221887:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764316.221888:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764316.221889:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764316.221890:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764316.221891:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764316.221892:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f6800.
02000000:00000001:3.0:1787764316.221893:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764316.221894:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764316.221895:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:3.0:1787764316.221896:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764316.221896:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.221897:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764316.221899:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832712960/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764341 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764316.221903:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764316.221903:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764316.221904:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764316.221906:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764316.221906:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764316.221907:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764316.221908:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764316.221909:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f6c00.
02000000:00000001:3.0:1787764316.221910:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764316.221911:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764316.221911:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:3.0:1787764316.221913:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764316.221914:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.221915:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764316.221918:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764316.221919:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764316.221920:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764316.221922:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764316.860210:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764316.860216:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764316.860221:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764316.860222:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764316.860226:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764316.860227:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764316.860240:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764316.860247:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764316.860251:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764316.860256:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764316.860260:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1787764316.860264:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000100:00000001:2.0:1787764316.860265:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1787764316.860270:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764316.860270:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764316.860273:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00010000:0.0:1787764316.860275:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:2.0:1787764316.860277:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764317.244240:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764317.244247:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764317.244249:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764317.244253:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764317.244257:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764317.244260:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764317.244263:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764317.244266:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764317.244270:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764317.500227:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316704499, at 4316704512
00000100:00000010:3.0:1787764317.500244:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40d80.
00000100:00000040:2.0:1787764317.500263:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316704499, at 4316704512
00000100:00000010:2.0:1787764317.500273:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40a80.
00000100:00000001:2.0:1787764317.884229:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764317.884230:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764317.884235:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764317.884238:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764317.884242:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764317.884242:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764317.884247:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00100000:2.0:1787764317.884249:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764317.884251:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764317.884254:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764317.884255:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764317.884259:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764317.884260:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764317.884261:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764317.884266:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764317.884266:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764317.884271:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764317.884271:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764317.884278:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764317.884279:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764317.884281:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764317.884327:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764317.884330:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764317.884333:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764318.268230:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764318.268237:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764318.268239:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764318.268244:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764318.268247:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764318.268250:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764318.268252:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764318.268254:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764318.268257:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764318.908217:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764318.908223:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764318.908226:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764318.908227:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764318.908228:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764318.908233:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764318.908233:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764318.908234:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764318.908237:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764318.908238:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764318.908239:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764318.908240:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764318.908242:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764318.908246:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764318.908248:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764318.908276:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764318.908279:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764318.908282:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764319.292217:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764319.292225:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764319.292227:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764319.292233:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764319.292237:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764319.292241:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764319.292244:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764319.292246:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764319.292250:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764319.932236:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764319.932239:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764319.932243:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764319.932246:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764319.932249:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764319.932251:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764319.932251:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764319.932257:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764319.932259:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764319.932260:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764319.932266:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764319.932270:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764319.932273:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764319.932277:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764319.932280:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764319.932297:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764319.932301:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764319.932304:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764320.316242:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764320.316250:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764320.316253:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764320.316259:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764320.316263:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764320.316268:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764320.316272:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764320.316275:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764320.316278:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764320.956209:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764320.956215:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764320.956216:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764320.956220:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764320.956221:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764320.956226:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764320.956230:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764320.956231:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764320.956234:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764320.956235:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764320.956239:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764320.956240:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764320.956241:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764320.956245:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764320.956246:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764320.956305:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764320.956308:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764320.956310:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764321.340235:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764321.340242:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764321.340244:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764321.340249:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764321.340252:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764321.340255:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.340257:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764321.340259:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.340261:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764321.340329:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764321.340334:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787764321.340336:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764321.340339:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764321.340343:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764321.340350:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 22035
00000400:00000200:2.0:1787764321.340357:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764321.340363:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764321.340366:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764321.340369:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764321.340372:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764321.340374:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764321.340376:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764321.340379:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764321.340381:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787764321.340383:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764321.340386:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787764321.340387:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787764321.340389:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787764321.340391:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764321.340392:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764321.340394:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764321.340395:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764321.340397:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787764321.340399:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764321.340400:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764321.340402:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764321.340405:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764321.340407:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764321.340408:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764321.340410:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764321.340417:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658780.
00000020:00000040:2.0:1787764321.340419:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787764321.340422:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764321.340423:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764321.340425:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764321.340427:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764321.340430:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764321.340436:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f7400.
00000100:00000001:2.0:1787764321.340442:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764321.340449:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658780 x1874599832713088/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764321.340456:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764321.340460:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764321.340463:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000744875b5] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658780 x1874599832713088/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764321.340481:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764321.340484:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764321.340486:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787764321.340487:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764321.340488:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764321.340490:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764321.340492:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22035
00000400:00000200:2.0:1787764321.340495:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764321.340498:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764321.340500:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787764321.340501:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:2.0:1787764321.340502:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764321.340505:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764321.340506:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764321.340509:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764321.340511:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764321.340514:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:3.0:1787764321.340517:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832713088/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:2.0:1787764321.340518:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764321.340522:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:2.0:1787764321.340526:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787764321.340528:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000040:2.0:1787764321.340531:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000001:3.0:1787764321.340531:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764321.340536:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000020:00000001:2.0:1787764321.340537:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:3.0:1787764321.340539:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:2.0:1787764321.340544:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:3.0:1787764321.340544:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764321.340549:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832713088:192.168.203.110@tcp:250:kworker.0
00000100:00000040:2.0:1787764321.340554:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764321.340555:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1787764321.340559:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1787764321.340560:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764321.340563:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764321.340566:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:3.0:1787764321.340567:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:2.0:1787764321.340571:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000010:3.0:1787764321.340572:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
00000020:00000040:2.0:1787764321.340574:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:3.0:1787764321.340575:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764321.340580:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:3.0:1787764321.340584:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764321.340585:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764321.340589:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:3.0:1787764321.340589:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
02000000:00000001:2.0:1787764321.340594:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:3.0:1787764321.340596:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
02000000:00000001:2.0:1787764321.340598:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764321.340601:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000200:3.0:1787764321.340603:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832713088, portal 25
00000008:00100000:2.0:1787764321.340605:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787764321.340606:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000008:00000001:2.0:1787764321.340609:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764321.340610:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000100:00000001:2.0:1787764321.340613:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764321.340618:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832713088/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764346 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1787764321.340619:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000020:00000040:2.0:1787764321.340624:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764321.340630:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1787764321.340630:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:2.0:1787764321.340632:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764321.340635:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787764321.340637:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764321.340640:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000010:3.0:1787764321.340640:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
02000000:00000001:2.0:1787764321.340643:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764321.340644:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832713088, offset 0
02000000:00000010:2.0:1787764321.340650:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f4800.
00000400:00000200:3.0:1787764321.340652:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787764321.340658:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764321.340661:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764321.340669:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00100000:2.0:1787764321.340670:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832713216/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787764321.340673:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764321.340679:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764321.340684:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764321.340689:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:2.0:1787764321.340690:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1787764321.340694:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58806:58806:256:4294967295] 192.168.203.110@tcp LPNI seq info [58806:58806:8:4294967295]
00000100:00000001:2.0:1787764321.340696:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764321.340700:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:2.0:1787764321.340707:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832713216/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764321.340711:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764321.340722:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764321.340728:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000100:00000001:2.0:1787764321.340731:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787764321.340737:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764321.340740:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787764321.340744:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764321.340747:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000100:00080000:2.0:1787764321.340749:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000800:00000100:3.0:1787764321.340751:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764321.340754:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000100:00080000:2.0:1787764321.340758:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000200:3.0:1787764321.340761:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1787764321.340762:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764321.340766:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1787764321.340767:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1787764321.340772:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:3.0:1787764321.340772:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764321.340774:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00080000:2.0:1787764321.340778:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22035
00000100:00000001:3.0:1787764321.340779:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764321.340784:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832713088/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764346 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764321.340786:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764321.340793:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764321.340797:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787764321.340799:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764321.340803:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000040:3.0:1787764321.340805:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832713088/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764346 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764321.340809:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764321.340814:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764321.340817:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764321.340817:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:2.0:1787764321.340823:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764321.340823:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764321.340829:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:3.0:1787764321.340830:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:2.0:1787764321.340834:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000100:00000001:3.0:1787764321.340834:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316709619
00000020:00000001:2.0:1787764321.340839:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000010:3.0:1787764321.340841:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40d80.
00000020:00000040:2.0:1787764321.340844:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:2.0:1787764321.340849:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:2.0:1787764321.340852:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000001:3.0:1787764321.340853:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000040:2.0:1787764321.340858:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764321.340860:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316709619
00000100:00000001:2.0:1787764321.340862:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000010:3.0:1787764321.340864:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d402a0.
00000100:00000040:2.0:1787764321.340868:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764321.340869:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787764321.340873:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:2.0:1787764321.340875:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764321.340876:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764321.340880:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000020:00000040:2.0:1787764321.340882:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00100000:3.0:1787764321.340885:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764321/real 1787764321]  req@ffff9751f8658780 x1874599832713088/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787764321.340890:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000001:3.0:1787764321.340897:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000001:2.0:1787764321.340898:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1787764321.340899:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
02000000:00000001:2.0:1787764321.340902:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000010:3.0:1787764321.340903:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764321.340907:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
02000000:00000001:2.0:1787764321.340908:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000010:3.0:1787764321.340908:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764321.340913:0:257575:0:(events.c:78:reply_in_callback()) Process entered
02000000:00000001:2.0:1787764321.340914:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764321.340917:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000200:3.0:1787764321.340919:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832713088/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:2.0:1787764321.340920:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000200:3.0:1787764321.340930:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832713088/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:2.0:1787764321.340933:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764321.340935:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764321.340938:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764321.340939:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764321.340940:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764321.340942:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764321.340946:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832713088/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1787764321.340948:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000100:00000001:3.0:1787764321.340955:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000020:00000040:2.0:1787764321.340956:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000040:3.0:1787764321.340960:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832713088/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764321.340962:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764321.340966:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764321.340967:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:2.0:1787764321.340972:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1787764321.340972:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.340975:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:2.0:1787764321.340977:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1787764321.340977:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.340981:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
02000000:00000001:2.0:1787764321.340983:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764321.340983:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764321.340989:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764321.340992:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
02000000:00000010:2.0:1787764321.340993:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f7c00.
00000100:00000001:3.0:1787764321.340994:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764321.340999:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832713088/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764321.341002:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764321.341020:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832713088:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764321.341025:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:2.0:1787764321.341028:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832713344/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764321.341028:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764321.341033:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832713088/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341056:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
00000100:00000001:2.0:1787764321.341060:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
02000000:00000001:3.0:1787764321.341060:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764321.341062:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
00000100:00000001:2.0:1787764321.341065:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764321.341067:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764321.341071:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764321.341072:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764321.341075:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
00000100:00000040:2.0:1787764321.341077:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832713344/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1787764321.341078:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f7400.
02000000:00000001:3.0:1787764321.341082:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764321.341084:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764321.341086:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:3.0:1787764321.341090:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764321.341091:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.341093:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.341098:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764321.341100:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764321.341102:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764321.341103:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764321.341106:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764321.341108:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:2.0:1787764321.341109:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1787764321.341114:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832713216/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787764321.341115:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:2.0:1787764321.341120:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764321.341128:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:3.0:1787764321.341131:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764321.341132:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1787764321.341150:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764321.341152:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764321.341154:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764321.341157:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764321.341159:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764321.341164:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00100000:3.0:1787764321.341164:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832713216:192.168.203.110@tcp:38:kworker.0
00000100:00080000:2.0:1787764321.341170:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22035
00000100:00000001:3.0:1787764321.341173:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000400:00000200:2.0:1787764321.341176:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:3.0:1787764321.341178:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000400:00000200:2.0:1787764321.341181:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
02000000:00000001:3.0:1787764321.341182:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764321.341184:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
02000000:00000001:3.0:1787764321.341187:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:2.0:1787764321.341190:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000010:3.0:1787764321.341192:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
00000100:00000001:2.0:1787764321.341193:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764321.341197:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:3.0:1787764321.341197:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764321.341202:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:3.0:1787764321.341207:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:2.0:1787764321.341208:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764321.341210:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787764321.341213:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000010:3.0:1787764321.341213:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000020:00000040:2.0:1787764321.341216:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787764321.341219:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000010:3.0:1787764321.341220:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000020:00000001:2.0:1787764321.341223:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000200:3.0:1787764321.341226:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832713216, portal 10
00000100:00000040:2.0:1787764321.341227:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764321.341232:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764321.341232:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764321.341235:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:2.0:1787764321.341237:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764321.341240:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764321.341242:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000040:3.0:1787764321.341245:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832713216/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764346 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:2.0:1787764321.341247:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764321.341252:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764321.341256:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764321.341258:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:2.0:1787764321.341260:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1787764321.341262:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764321.341265:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
02000000:00000001:2.0:1787764321.341267:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764321.341268:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832713216, offset 0
00000100:00000001:2.0:1787764321.341273:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1787764321.341275:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000008:00100000:2.0:1787764321.341279:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000400:00000200:3.0:1787764321.341280:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000008:00000001:2.0:1787764321.341286:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764321.341286:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764321.341290:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:2.0:1787764321.341292:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764321.341293:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764321.341298:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764321.341300:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000010:2.0:1787764321.341301:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000400:00000200:3.0:1787764321.341303:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58807:58807:256:4294967295] 192.168.203.110@tcp LPNI seq info [58807:58807:8:4294967295]
00000020:00000040:2.0:1787764321.341308:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000400:00000200:3.0:1787764321.341308:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:2.0:1787764321.341314:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:3.0:1787764321.341316:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
02000000:00000001:2.0:1787764321.341317:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787764321.341321:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787764321.341323:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000800:00000010:3.0:1787764321.341325:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
02000000:00000001:2.0:1787764321.341327:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000800:00000200:3.0:1787764321.341331:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000001:2.0:1787764321.341333:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787764321.341334:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787764321.341340:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000010:2.0:1787764321.341342:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f6400.
00000800:00000200:3.1:1787764321.341343:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000100:00000001:2.0:1787764321.341350:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1787764321.341351:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764321.341356:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764321.341360:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00100000:2.0:1787764321.341361:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832713472/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787764321.341368:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764321.341375:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000100:00000001:2.0:1787764321.341376:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764321.341380:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764321.341380:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764321.341386:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:2.0:1787764321.341388:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832713472/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764321.341394:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832713216/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764346 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764321.341399:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764321.341406:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (22046)
00000100:00000001:3.0:1787764321.341407:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764321.341411:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832713216/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764346 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341416:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764321.341418:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764321.341422:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764321.341424:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764321.341427:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764321.341429:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316708384
00000100:00000001:3.0:1787764321.341431:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316709619
00000100:00000001:3.0:1787764321.341432:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316708384
00000100:00000001:3.0:1787764321.341434:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316709619
00000100:00000001:3.0:1787764321.341435:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764321.341437:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764321.341438:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764321.341441:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764321/real 1787764321]  req@ffff9751f865ad00 x1874599832713216/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764321.341445:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764321.341446:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764321.341447:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764321.341449:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764321.341450:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764321.341451:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764321.341453:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832713216/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764321.341458:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832713216/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341462:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764321.341463:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.341464:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764321.341465:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764321.341467:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832713216/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341471:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764321.341474:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832713216/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341479:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764321.341481:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.341482:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764321.341482:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.341484:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764321.341493:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764321.341495:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764321.341496:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764321.341498:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764321.341500:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832713216/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764321.341506:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832713216:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764321.341508:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764321.341510:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832713344/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341514:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764321.341515:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764321.341516:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764321.341517:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764321.341519:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764321.341520:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832713344:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764321.341522:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764321.341523:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764321.341524:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764321.341525:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764321.341528:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764321.341529:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764321.341532:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764321.341533:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764321.341535:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764321.341537:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832713344, portal 10
00000100:00000001:3.0:1787764321.341538:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764321.341539:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:3.0:1787764321.341543:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832713344/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764346 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341547:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764321.341549:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764321.341550:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764321.341552:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832713344, offset 0
00000400:00000200:3.0:1787764321.341554:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764321.341556:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764321.341559:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764321.341561:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764321.341562:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764321.341564:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764321.341565:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764321.341567:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58808:58808:256:4294967295] 192.168.203.110@tcp LPNI seq info [58808:58808:8:4294967295]
00000400:00000200:3.0:1787764321.341570:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764321.341574:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764321.341577:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764321.341579:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764321.341582:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764321.341584:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787764321.341587:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764321.341588:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1787764321.341591:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764321.341592:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764321.341594:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764321.341596:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764321.341598:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764321.341600:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764321.341601:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764321.341604:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832713344/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764346 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341608:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764321.341610:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832713344/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764346 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341614:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764321.341615:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764321.341617:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764321.341619:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764321.341621:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764321.341622:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316708372
00000100:00000001:3.0:1787764321.341624:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316709619
00000100:00000001:3.0:1787764321.341625:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316708372
00000100:00000001:3.0:1787764321.341626:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316709619
00000100:00000001:3.0:1787764321.341628:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764321.341629:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764321.341630:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764321.341632:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764321/real 1787764321]  req@ffff9751f86583c0 x1874599832713344/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764321.341636:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764321.341637:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764321.341638:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764321.341640:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764321.341641:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764321.341642:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764321.341644:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832713344/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764321.341649:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832713344/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341653:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764321.341654:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.341656:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764321.341657:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764321.341659:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832713344/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341663:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764321.341665:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832713344/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341669:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764321.341670:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.341671:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764321.341672:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.341673:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764321.341674:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764321.341675:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764321.341677:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764321.341678:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764321.341681:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832713344/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764321.341685:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832713344:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764321.341688:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.341689:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764321.341691:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832713216/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341695:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764321.341697:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764321.341698:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764321.341700:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764321.341701:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764321.341702:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764321.341703:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764321.341704:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f4800.
02000000:00000001:3.0:1787764321.341706:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764321.341707:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764321.341708:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:3.0:1787764321.341710:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764321.341711:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.341712:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764321.341713:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832713344/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341718:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764321.341719:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764321.341720:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764321.341721:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764321.341722:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764321.341723:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764321.341724:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764321.341725:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f7c00.
02000000:00000001:3.0:1787764321.341726:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764321.341727:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764321.341728:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:3.0:1787764321.341729:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764321.341730:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.341731:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.341733:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764321.341734:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764321.341735:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764321.341736:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764321.341737:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764321.341739:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832713472/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341743:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764321.341744:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764321.341745:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764321.341746:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764321.341748:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764321.341750:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832713472:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764321.341751:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764321.341753:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764321.341753:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764321.341755:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764321.341756:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764321.341757:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764321.341759:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764321.341761:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764321.341763:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764321.341765:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832713472, portal 4
00000100:00000001:3.0:1787764321.341766:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764321.341767:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:3.0:1787764321.341770:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832713472/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764346 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341774:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764321.341776:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764321.341777:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764321.341779:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832713472, offset 0
00000400:00000200:3.0:1787764321.341781:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764321.341783:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764321.341785:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764321.341786:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764321.341788:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764321.341789:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764321.341791:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764321.341793:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58809:58809:256:4294967295] 192.168.203.110@tcp LPNI seq info [58809:58809:8:4294967295]
00000400:00000200:3.0:1787764321.341796:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764321.341800:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764321.341803:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764321.341805:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764321.341808:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764321.341810:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000200:3.1:1787764321.341812:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764321.341814:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 32 secs later)
00000800:00000100:3.0:1787764321.341816:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764321.341818:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764321.341819:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764321.341822:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764321.341823:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764321.341824:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764321.341826:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764321.341828:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832713472/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764346 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341833:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764321.341835:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832713472/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764346 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341839:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764321.341840:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764321.341841:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764321.341843:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764321.341845:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764321.341846:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316708389
00000100:00000001:3.0:1787764321.341848:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316709619
00000100:00000001:3.0:1787764321.341849:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316708389
00000100:00000001:3.0:1787764321.341850:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316709619
00000100:00000001:3.0:1787764321.341852:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764321.341853:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764321.341854:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764321.341856:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764321/real 1787764321]  req@ffff9751f8659a40 x1874599832713472/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764321.341862:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764321.341862:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764321.341864:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764321.341865:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764321.341866:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764321.341867:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764321.341869:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832713472/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764321.341874:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832713472/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341877:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764321.341878:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.341880:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764321.341881:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764321.341883:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832713472/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341887:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764321.341889:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832713472/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341893:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764321.341895:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.341896:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764321.341896:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.341897:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764321.341898:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764321.341900:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764321.341902:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764321.341903:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764321.341906:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832713472/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764321.341911:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832713472:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764321.341913:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.341914:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764321.341916:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832713472/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764346 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764321.341920:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764321.341921:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764321.341922:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764321.341923:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764321.341923:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764321.341924:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764321.341925:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764321.341926:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f6400.
02000000:00000001:3.0:1787764321.341928:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764321.341928:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764321.341929:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:3.0:1787764321.341931:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764321.341931:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.341932:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764321.341934:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764321.341935:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764321.341937:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764321.341938:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764321.980240:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764321.980241:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764321.980246:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764321.980248:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764321.980252:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764321.980253:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764321.980256:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764321.980260:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764321.980261:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764321.980264:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764321.980266:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764321.980279:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764321.980282:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764321.980285:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764321.980288:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764321.980326:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764321.980330:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764321.980333:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764322.364244:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764322.364253:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764322.364255:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764322.364261:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764322.364265:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764322.364269:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764322.364271:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764322.364274:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764322.364276:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764322.620229:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316709619, at 4316709632
00000100:00000010:3.0:1787764322.620243:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d402a0.
00000100:00000040:2.0:1787764322.620261:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316709619, at 4316709632
00000100:00000010:2.0:1787764322.620269:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40d80.
00000100:00000001:2.0:1787764323.004210:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764323.004217:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764323.004219:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764323.004223:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764323.004227:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764323.004229:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764323.004232:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764323.004234:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764323.004236:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764323.004334:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764323.004338:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764323.004342:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764323.004343:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764323.004347:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764323.004348:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764323.004396:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764323.004401:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764323.004405:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764323.388275:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764323.388283:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764323.388286:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764323.388292:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764323.388297:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764323.388301:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764323.388304:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764323.388307:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764323.388310:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764324.028236:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764324.028242:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764324.028244:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764324.028246:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764324.028247:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764324.028253:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764324.028253:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764324.028256:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764324.028258:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764324.028261:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764324.028263:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764324.028264:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764324.028268:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764324.028270:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764324.028271:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764324.028275:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764324.028275:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764324.028277:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764324.028280:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764324.028281:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764324.028282:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764324.028339:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764324.028343:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764324.028345:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764324.412218:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764324.412227:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764324.412231:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764324.412236:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764324.412240:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764324.412244:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764324.412247:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764324.412249:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764324.412253:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764325.052243:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764325.052251:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764325.052252:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764325.052257:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764325.052258:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764325.052267:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764325.052267:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764325.052271:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764325.052273:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764325.052278:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764325.052279:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764325.052281:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764325.052285:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764325.052291:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764325.052294:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764325.052349:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764325.052354:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764325.052358:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764325.436239:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764325.436245:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764325.436247:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764325.436252:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764325.436256:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764325.436260:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764325.436262:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764325.436264:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764325.436266:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764326.076237:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764326.076243:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764326.076244:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764326.076248:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764326.076251:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00100000:2.0:1787764326.076254:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764326.076256:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764326.076259:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764326.076261:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764326.076263:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764326.076269:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764326.076269:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764326.076273:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764326.076274:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764326.076276:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764326.076279:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764326.076284:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764326.076289:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764326.076291:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764326.076295:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764326.076297:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764326.076334:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764326.076338:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764326.076341:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:2.0:1787764326.460220:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764326.460224:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00080000:2.0:1787764326.460229:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764326.460230:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764326.460233:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1787764326.460234:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764326.460237:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00100000:3.0:1787764326.460238:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:2.0:1787764326.460242:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764326.460244:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764326.460248:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1787764326.460249:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 22041
00000100:00000001:3.0:1787764326.460251:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764326.460254:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764326.460257:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:2.0:1787764326.460258:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764326.460264:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764326.460267:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764326.460270:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764326.460272:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764326.460275:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764326.460277:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764326.460280:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764326.460281:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787764326.460283:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764326.460286:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787764326.460287:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787764326.460289:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787764326.460291:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764326.460293:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764326.460294:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764326.460296:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764326.460298:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787764326.460301:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764326.460303:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764326.460305:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764326.460307:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764326.460309:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764326.460311:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764326.460312:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764326.460320:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
00000020:00000040:2.0:1787764326.460322:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787764326.460325:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764326.460326:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764326.460328:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764326.460330:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764326.460333:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764326.460339:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f6800.
00000100:00000001:2.0:1787764326.460345:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764326.460352:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832713600/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764326.460360:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764326.460365:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764326.460371:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832713600/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764326.460391:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764326.460395:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764326.460399:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787764326.460401:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764326.460403:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764326.460411:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764326.460416:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764326.460417:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764326.460418:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00080000:2.0:1787764326.460422:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22041
00000100:00000040:3.0:1787764326.460427:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832713600/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1787764326.460432:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764326.460440:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000001:3.0:1787764326.460441:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764326.460443:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764326.460446:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
02000000:00000001:3.0:1787764326.460447:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764326.460451:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:2.0:1787764326.460454:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:3.0:1787764326.460455:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764326.460461:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:3.0:1787764326.460462:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832713600:192.168.203.110@tcp:250:kworker.0
00000100:00000040:2.0:1787764326.460467:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764326.460468:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764326.460472:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000001:2.0:1787764326.460473:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:3.0:1787764326.460475:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764326.460479:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000001:2.0:1787764326.460481:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764326.460484:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
02000000:00000010:3.0:1787764326.460485:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
00000020:00000001:2.0:1787764326.460487:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
02000000:00000001:3.0:1787764326.460488:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:2.0:1787764326.460493:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000400:00000200:3.0:1787764326.460501:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764326.460505:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000020:00000001:2.0:1787764326.460507:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:2.0:1787764326.460512:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000400:00000010:3.0:1787764326.460513:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000040:2.0:1787764326.460517:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000200:3.0:1787764326.460522:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832713600, portal 25
00000100:00000001:2.0:1787764326.460528:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764326.460529:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764326.460533:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000100:00000040:2.0:1787764326.460534:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764326.460539:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764326.460541:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:2.0:1787764326.460547:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764326.460549:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764326.460551:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:3.0:1787764326.460555:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832713600/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764351 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764326.460560:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764326.460569:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:2.0:1787764326.460572:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764326.460578:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:3.0:1787764326.460578:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764326.460584:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000008:00100000:2.0:1787764326.460585:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787764326.460591:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764326.460591:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832713600, offset 0
00000100:00000001:2.0:1787764326.460596:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764326.460604:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000400:00000200:3.0:1787764326.460604:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000020:00000040:2.0:1787764326.460609:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764326.460613:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764326.460615:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764326.460618:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1787764326.460619:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:2.0:1787764326.460620:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764326.460624:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764326.460628:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f5000.
00000400:00000200:3.0:1787764326.460629:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000001:2.0:1787764326.460633:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764326.460636:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00100000:2.0:1787764326.460641:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832713728/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787764326.460642:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764326.460648:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764326.460653:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:2.0:1787764326.460656:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1787764326.460659:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58810:58810:256:4294967295] 192.168.203.110@tcp LPNI seq info [58810:58810:8:4294967295]
00000100:00000001:2.0:1787764326.460661:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764326.460669:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832713728/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764326.460669:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764326.460680:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764326.460683:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00080000:2.0:1787764326.460685:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764326.460691:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1787764326.460694:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000800:00000200:3.0:1787764326.460695:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00080000:2.0:1787764326.460697:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000800:00000010:3.0:1787764326.460701:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764326.460708:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787764326.460709:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764326.460715:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22041
00000800:00000200:3.0:1787764326.460718:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000400:00000200:2.0:1787764326.460721:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:3.1:1787764326.460730:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000400:00000200:2.0:1787764326.460731:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764326.460736:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000200:3.1:1787764326.460738:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000100:00000040:2.0:1787764326.460742:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764326.460747:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000100:3.0:1787764326.460750:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764326.460754:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000100:00000040:2.0:1787764326.460760:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:3.0:1787764326.460763:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:2.0:1787764326.460764:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764326.460770:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764326.460773:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000400:00000200:3.0:1787764326.460774:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:2.0:1787764326.460777:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764326.460782:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000400:00000200:3.0:1787764326.460782:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000020:00000001:2.0:1787764326.460786:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000010:3.0:1787764326.460787:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000020:00000001:2.0:1787764326.460792:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000001:3.0:1787764326.460795:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:2.0:1787764326.460799:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764326.460802:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764326.460806:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000200:3.0:1787764326.460806:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832713600/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764351 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764326.460809:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764326.460814:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:2.0:1787764326.460818:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764326.460823:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764326.460827:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764326.460831:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764326.460831:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000001:2.0:1787764326.460834:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764326.460838:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000040:3.0:1787764326.460839:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832713600/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764351 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00100000:2.0:1787764326.460840:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787764326.460846:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764326.460850:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764326.460856:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000100:00000001:3.0:1787764326.460856:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764326.460860:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000020:00000040:2.0:1787764326.460861:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764326.460865:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764326.460866:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764326.460868:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1787764326.460868:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764326.460872:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764326.460874:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764326.460875:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764326.460880:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316714739
02000000:00000010:2.0:1787764326.460881:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f4000.
00000100:00000001:2.0:1787764326.460886:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764326.460887:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d402a0.
00000100:00100000:2.0:1787764326.460894:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832713856/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764326.460895:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787764326.460903:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316714739
00000100:00000001:2.0:1787764326.460906:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000010:3.0:1787764326.460907:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d408a0.
00000100:00000001:3.0:1787764326.460911:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:2.0:1787764326.460912:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764326.460914:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764326.460918:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764326.460921:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00000040:2.0:1787764326.460923:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832713856/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764326.460927:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764326/real 1787764326]  req@ffff9751f8658000 x1874599832713600/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764326.460954:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1787764326.460955:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764326.460957:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000040:2.0:1787764326.460959:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:3.0:1787764326.460960:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764326.460965:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000100:00000040:2.0:1787764326.460966:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000010:3.0:1787764326.460968:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00080000:2.0:1787764326.460985:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764326.460985:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00080000:2.0:1787764326.460990:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000200:3.0:1787764326.460992:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832713600/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764326.460993:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764326.460995:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764326.461000:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764326.461005:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22041
00000400:00000200:2.0:1787764326.461012:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764326.461018:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764326.461020:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000200:3.0:1787764326.461020:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832713600/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787764326.461025:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764326.461029:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764326.461031:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764326.461035:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764326.461039:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764326.461042:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787764326.461043:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787764326.461043:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000020:00000040:2.0:1787764326.461047:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000001:3.0:1787764326.461048:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:2.0:1787764326.461050:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:3.0:1787764326.461051:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000020:00000001:2.0:1787764326.461053:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:3.0:1787764326.461054:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:2.0:1787764326.461058:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764326.461062:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00100000:3.0:1787764326.461063:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832713600/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:2.0:1787764326.461066:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764326.461070:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764326.461074:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:2.0:1787764326.461079:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764326.461080:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764326.461082:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764326.461085:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:3.0:1787764326.461087:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832713600/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764326.461089:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764326.461093:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764326.461097:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764326.461101:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000001:3.0:1787764326.461102:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000008:00000001:2.0:1787764326.461107:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764326.461108:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764326.461111:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764326.461111:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764326.461115:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000010:2.0:1787764326.461117:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000100:00000001:3.0:1787764326.461118:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764326.461121:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000020:00000040:2.0:1787764326.461122:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00080000:3.0:1787764326.461125:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
02000000:00000001:2.0:1787764326.461126:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764326.461128:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764326.461130:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:2.0:1787764326.461131:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764326.461152:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764326.461154:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764326.461158:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f4c00.
00000100:00000001:3.0:1787764326.461158:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:2.0:1787764326.461164:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764326.461167:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832713600/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:2.0:1787764326.461172:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832713984/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00100000:3.0:1787764326.461183:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832713600:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764326.461187:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764326.461189:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764326.461190:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764326.461194:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:2.0:1787764326.461197:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832713984/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787764326.461201:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832713600/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764326.461218:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764326.461225:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
00000100:00000040:2.0:1787764326.461226:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (22051)
02000000:00000001:3.0:1787764326.461231:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764326.461235:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764326.461241:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764326.461243:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764326.461245:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764326.461247:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764326.461251:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f6800.
02000000:00000001:3.0:1787764326.461254:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764326.461256:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764326.461258:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:3.0:1787764326.461261:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764326.461263:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764326.461265:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764326.461272:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764326.461274:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764326.461276:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764326.461278:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764326.461280:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764326.461284:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832713728/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.461292:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764326.461294:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764326.461296:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764326.461298:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764326.461301:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764326.461304:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832713728:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764326.461308:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764326.461310:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764326.461312:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764326.461314:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764326.461318:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764326.461320:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764326.461324:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764326.461327:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764326.461332:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764326.461335:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832713728, portal 10
00000100:00000001:3.0:1787764326.461338:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764326.461339:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:3.0:1787764326.461346:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832713728/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764351 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.461353:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764326.461356:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764326.461358:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764326.461361:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832713728, offset 0
00000400:00000200:3.0:1787764326.461366:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764326.461370:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764326.461375:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764326.461378:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764326.461381:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764326.461384:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764326.461386:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764326.461389:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58811:58811:256:4294967295] 192.168.203.110@tcp LPNI seq info [58811:58811:8:4294967295]
00000400:00000200:3.0:1787764326.461394:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764326.461401:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764326.461406:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764326.461409:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764326.461414:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764326.461417:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787764326.461422:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764326.461425:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1787764326.461429:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764326.461431:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764326.461435:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764326.461439:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764326.461442:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764326.461444:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764326.461447:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764326.461451:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832713728/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764351 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.461460:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764326.461463:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832713728/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764351 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.461470:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764326.461471:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764326.461475:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764326.461480:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764326.461484:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764326.461486:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316713504
00000100:00000001:3.0:1787764326.461489:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316714739
00000100:00000001:3.0:1787764326.461491:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316713504
00000100:00000001:3.0:1787764326.461495:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316714739
00000100:00000001:3.0:1787764326.461497:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764326.461499:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764326.461501:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764326.461505:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764326/real 1787764326]  req@ffff9751f8659a40 x1874599832713728/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764326.461514:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764326.461516:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764326.461517:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764326.461520:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764326.461522:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764326.461524:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764326.461528:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832713728/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764326.461538:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832713728/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.461545:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764326.461547:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764326.461549:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764326.461551:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764326.461555:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832713728/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.461562:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764326.461566:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832713728/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.461573:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764326.461576:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764326.461578:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764326.461579:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764326.461581:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764326.461594:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764326.461597:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764326.461601:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764326.461603:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764326.461608:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832713728/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764326.461617:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832713728:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764326.461621:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764326.461624:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832713856/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.461632:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764326.461633:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764326.461635:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764326.461638:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764326.461640:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764326.461643:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832713856:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764326.461646:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764326.461648:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764326.461650:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764326.461652:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764326.461656:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764326.461658:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764326.461662:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764326.461664:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764326.461668:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764326.461671:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832713856, portal 10
00000100:00000001:3.0:1787764326.461673:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764326.461674:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:3.0:1787764326.461681:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832713856/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764351 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.461688:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764326.461691:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764326.461694:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764326.461697:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832713856, offset 0
00000400:00000200:3.0:1787764326.461701:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764326.461705:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764326.461709:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764326.461712:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764326.461715:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764326.461717:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764326.461720:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764326.461723:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58812:58812:256:4294967295] 192.168.203.110@tcp LPNI seq info [58812:58812:8:4294967295]
00000400:00000200:3.0:1787764326.461728:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764326.461734:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764326.461739:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764326.461743:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764326.461748:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764326.461751:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787764326.461755:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764326.461759:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1787764326.461763:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764326.461765:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764326.461768:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764326.461771:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764326.461775:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764326.461777:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764326.461779:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764326.461783:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832713856/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764351 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.461791:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764326.461795:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832713856/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764351 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.461808:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764326.461810:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764326.461813:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764326.461816:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764326.461820:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764326.461822:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316713492
00000100:00000001:3.0:1787764326.461824:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316714739
00000100:00000001:3.0:1787764326.461827:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316713492
00000100:00000001:3.0:1787764326.461829:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316714739
00000100:00000001:3.0:1787764326.461831:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764326.461833:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764326.461836:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764326.461841:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764326/real 1787764326]  req@ffff9751f86583c0 x1874599832713856/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764326.461850:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764326.461852:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764326.461853:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764326.461856:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764326.461858:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764326.461860:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764326.461864:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832713856/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764326.461873:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832713856/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.461880:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764326.461881:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764326.461883:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764326.461885:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764326.461889:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832713856/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.461897:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764326.461902:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832713856/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.461910:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764326.461913:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764326.461915:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764326.461916:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764326.461918:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764326.461920:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764326.461922:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764326.461926:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764326.461928:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764326.461932:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832713856/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764326.461941:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832713856:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764326.461945:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764326.461948:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832713984/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.461956:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764326.461958:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764326.461960:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764326.461962:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764326.461965:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764326.461968:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832713984:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764326.461988:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764326.461990:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764326.461992:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764326.461994:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764326.461998:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764326.462000:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764326.462004:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764326.462006:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764326.462010:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764326.462013:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832713984, portal 4
00000100:00000001:3.0:1787764326.462015:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764326.462016:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:3.0:1787764326.462022:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832713984/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764351 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.462030:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764326.462033:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764326.462035:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764326.462037:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832713984, offset 0
00000400:00000200:3.0:1787764326.462041:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764326.462045:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764326.462049:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764326.462052:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764326.462055:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764326.462057:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764326.462060:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764326.462064:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58813:58813:256:4294967295] 192.168.203.110@tcp LPNI seq info [58813:58813:8:4294967295]
00000400:00000200:3.0:1787764326.462069:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764326.462075:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764326.462079:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764326.462083:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764326.462087:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764326.462090:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000200:3.1:1787764326.462095:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764326.462098:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 27 secs later)
00000800:00000100:3.0:1787764326.462102:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764326.462105:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764326.462108:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764326.462112:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764326.462116:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764326.462117:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764326.462120:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764326.462125:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832713984/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764351 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.462153:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764326.462158:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832713984/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764351 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.462166:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764326.462168:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764326.462172:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764326.462175:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764326.462179:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764326.462181:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316713510
00000100:00000001:3.0:1787764326.462184:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316714739
00000100:00000001:3.0:1787764326.462186:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316713510
00000100:00000001:3.0:1787764326.462189:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316714739
00000100:00000001:3.0:1787764326.462192:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764326.462194:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764326.462196:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764326.462200:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764326/real 1787764326]  req@ffff9751f865ad00 x1874599832713984/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764326.462209:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764326.462211:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764326.462213:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764326.462215:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764326.462217:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764326.462220:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764326.462224:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832713984/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764326.462233:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832713984/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.462240:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764326.462241:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764326.462244:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764326.462245:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764326.462249:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832713984/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.462257:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764326.462261:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832713984/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.462268:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764326.462271:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764326.462272:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764326.462274:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764326.462276:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764326.462278:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764326.462280:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764326.462283:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764326.462286:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764326.462290:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832713984/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764326.462298:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832713984:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764326.462301:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764326.462304:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764326.462308:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832713728/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.462315:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764326.462316:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764326.462318:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764326.462321:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764326.462323:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764326.462324:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764326.462326:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764326.462328:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f5000.
02000000:00000001:3.0:1787764326.462331:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764326.462333:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764326.462334:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:3.0:1787764326.462337:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764326.462338:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764326.462340:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764326.462343:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832713856/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.462350:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764326.462351:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764326.462353:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764326.462355:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764326.462356:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764326.462358:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764326.462360:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764326.462361:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f4000.
02000000:00000001:3.0:1787764326.462363:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764326.462364:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764326.462366:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:3.0:1787764326.462369:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764326.462370:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764326.462372:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764326.462375:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832713984/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764351 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764326.462381:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764326.462383:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764326.462385:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764326.462387:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764326.462388:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764326.462389:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764326.462391:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764326.462393:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f4c00.
02000000:00000001:3.0:1787764326.462395:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764326.462397:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764326.462398:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:3.0:1787764326.462401:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764326.462402:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764326.462404:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764326.462408:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764326.462410:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764326.462412:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764326.462413:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764327.100268:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764327.100277:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764327.100280:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764327.100286:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764327.100290:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764327.100294:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764327.100298:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764327.100299:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764327.100303:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764327.100304:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764327.100311:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764327.100312:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764327.100315:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764327.100317:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764327.100320:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764327.100371:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764327.100375:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764327.100378:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764327.484267:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764327.484286:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764327.484289:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764327.484299:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764327.484304:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764327.484308:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764327.484312:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764327.484315:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764327.484318:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764327.740252:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316714739, at 4316714752
00000100:00000010:3.0:1787764327.740267:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d408a0.
00000100:00000040:2.0:1787764327.740335:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316714739, at 4316714752
00000100:00000010:2.0:1787764327.740350:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d402a0.
00010000:00000001:3.0:1787764328.124262:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764328.124269:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764328.124273:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764328.124276:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764328.124284:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764328.124285:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764328.124290:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764328.124290:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00100000:2.0:1787764328.124300:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764328.124300:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764328.124312:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764328.124312:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764328.124318:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764328.124319:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764328.124327:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764328.124327:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764328.124330:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764328.124333:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764328.124335:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764328.124340:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764328.124343:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764328.124396:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764328.124400:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764328.124402:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764328.508346:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764328.508354:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764328.508357:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764328.508362:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764328.508370:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764328.508375:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764328.508378:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764328.508380:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764328.508383:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764329.148227:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764329.148227:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764329.148234:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764329.148236:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764329.148240:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764329.148241:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764329.148244:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764329.148245:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764329.148249:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764329.148250:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764329.148252:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764329.148255:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764329.148259:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764329.148262:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764329.148266:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764329.148277:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764329.148280:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764329.148283:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764329.532265:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764329.532273:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764329.532276:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764329.532282:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764329.532286:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764329.532289:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764329.532292:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764329.532294:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764329.532297:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764330.172229:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764330.172229:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764330.172235:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764330.172237:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764330.172241:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764330.172241:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764330.172245:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764330.172247:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764330.172251:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764330.172253:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764330.172254:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764330.172258:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764330.172262:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764330.172265:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764330.172268:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764330.172308:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764330.172312:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764330.172316:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764330.556220:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764330.556228:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764330.556231:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764330.556236:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764330.556240:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764330.556243:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764330.556246:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764330.556248:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764330.556251:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764331.196229:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764331.196232:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764331.196236:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764331.196238:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764331.196241:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764331.196241:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764331.196244:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764331.196246:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764331.196248:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764331.196251:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764331.196252:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764331.196255:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764331.196258:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764331.196260:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764331.196263:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764331.196304:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764331.196308:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764331.196310:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764331.580304:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764331.580314:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764331.580316:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764331.580321:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764331.580325:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764331.580329:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.580331:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764331.580334:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.580337:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764331.580572:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764331.580577:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:2.0:1787764331.580579:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764331.580582:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764331.580586:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764331.580592:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 22046
00000400:00000200:2.0:1787764331.580599:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764331.580605:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764331.580609:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764331.580612:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764331.580614:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764331.580617:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764331.580619:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764331.580623:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764331.580625:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787764331.580627:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764331.580630:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787764331.580633:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787764331.580635:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787764331.580637:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764331.580639:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764331.580640:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764331.580642:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764331.580645:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787764331.580648:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764331.580656:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764331.580658:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764331.580661:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764331.580664:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764331.580666:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764331.580669:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764331.580679:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658780.
00000020:00000040:2.0:1787764331.580682:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787764331.580685:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764331.580687:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764331.580690:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764331.580692:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764331.580696:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764331.580705:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f4c00.
00000100:00000001:2.0:1787764331.580712:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764331.580721:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658780 x1874599832714112/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764331.580731:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764331.580735:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764331.580740:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000744875b5] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658780 x1874599832714112/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764331.580757:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764331.580759:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764331.580762:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787764331.580763:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764331.580765:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764331.580768:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764331.580770:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22046
00000400:00000200:2.0:1787764331.580774:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764331.580777:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764331.580779:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764331.580782:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764331.580784:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764331.580786:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764331.580788:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764331.580790:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764331.580791:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:2.0:1787764331.580794:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764331.580796:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:2.0:1787764331.580798:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:2.0:1787764331.580800:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:2.0:1787764331.580803:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764331.580804:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764331.580806:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764331.580808:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764331.580810:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:2.0:1787764331.580812:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764331.580814:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764331.580816:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764331.580821:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764331.580823:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764331.580825:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764331.580828:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:2.0:1787764331.580832:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764331.580834:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764331.580838:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000020:00000040:2.0:1787764331.580841:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764331.580844:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764331.580845:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764331.580847:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764331.580848:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764331.580850:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764331.580855:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f4000.
00000100:00000001:2.0:1787764331.580859:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764331.580863:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832714240/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764331.580870:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764331.580872:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764331.580875:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832714240/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764331.580882:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764331.580885:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764331.580887:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1787764331.580888:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764331.580889:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764331.580891:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764331.580893:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22046
00000400:00000200:2.0:1787764331.580897:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764331.580899:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764331.580901:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764331.580904:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764331.580906:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764331.580908:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764331.580910:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764331.580911:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764331.580912:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:2.0:1787764331.580913:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764331.580915:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:2.0:1787764331.580917:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:2.0:1787764331.580919:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:2.0:1787764331.580922:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764331.580924:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764331.580925:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764331.580927:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764331.580929:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:2.0:1787764331.580932:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764331.580933:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764331.580946:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764331.580949:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764331.580951:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764331.580953:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764331.580954:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787764331.580955:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000008:00000001:2.0:1787764331.580957:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764331.580958:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764331.580958:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764331.580960:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000010:2.0:1787764331.580962:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000020:00000040:2.0:1787764331.580964:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764331.580966:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000040:3.0:1787764331.580966:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658780 x1874599832714112/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764331.580967:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764331.580969:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764331.580970:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764331.580972:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764331.580975:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f5000.
00000100:00000001:3.0:1787764331.580975:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764331.580976:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764331.580978:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764331.580980:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00100000:2.0:1787764331.580982:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832714368/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1787764331.580983:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764331.580986:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764331.580988:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764331.580990:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764331.580990:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832714112:192.168.203.110@tcp:250:kworker.0
00000100:00000040:2.0:1787764331.580994:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832714368/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.580998:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:2.0:1787764331.581002:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764331.581002:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764331.581004:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764331.581005:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:3.0:1787764331.581007:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:2.0:1787764331.581009:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
02000000:00000010:3.0:1787764331.581011:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
00000100:00080000:2.0:1787764331.581012:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:3.0:1787764331.581013:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764331.581015:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787764331.581016:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764331.581017:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1787764331.581019:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764331.581020:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000400:00000010:3.0:1787764331.581023:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000100:00080000:2.0:1787764331.581024:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22046
00000400:00000200:2.0:1787764331.581027:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764331.581030:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000010:3.0:1787764331.581031:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00080000:2.0:1787764331.581032:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764331.581035:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764331.581036:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:3.0:1787764331.581037:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832714112, portal 25
00000100:00000040:2.0:1787764331.581038:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764331.581040:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764331.581040:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764331.581041:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048389504 : -115096661162112 : ffff9751f8658780)
00000020:00000001:2.0:1787764331.581042:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764331.581043:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787764331.581045:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000040:3.0:1787764331.581046:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658780 x1874599832714112/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764356 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:2.0:1787764331.581047:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787764331.581049:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:2.0:1787764331.581051:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:2.0:1787764331.581054:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764331.581054:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:2.0:1787764331.581056:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764331.581057:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000040:3.0:1787764331.581057:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787764331.581059:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000010:3.0:1787764331.581059:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000020:00000040:2.0:1787764331.581061:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000200:3.0:1787764331.581063:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832714112, offset 0
00000100:00080000:2.0:1787764331.581064:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764331.581065:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764331.581067:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764331.581069:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1787764331.581069:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
02000000:00000001:2.0:1787764331.581071:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764331.581072:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764331.581075:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000400:00000200:3.0:1787764331.581076:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000008:00000001:2.0:1787764331.581078:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764331.581079:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764331.581082:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000400:00000200:3.0:1787764331.581082:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000020:00000040:2.0:1787764331.581084:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:2.0:1787764331.581087:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:3.0:1787764331.581088:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764331.581092:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
02000000:00000001:2.0:1787764331.581093:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764331.581095:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1787764331.581096:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
02000000:00000001:2.0:1787764331.581097:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764331.581098:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764331.581098:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
02000000:00000010:2.0:1787764331.581102:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f6800.
00000400:00000200:3.0:1787764331.581103:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58814:58814:256:4294967295] 192.168.203.110@tcp LPNI seq info [58814:58814:8:4294967295]
00000100:00000001:2.0:1787764331.581105:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764331.581107:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00100000:2.0:1787764331.581109:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832714496/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764331.581115:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1787764331.581120:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000001:2.0:1787764331.581121:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764331.581124:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832714496/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1787764331.581126:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:2.0:1787764331.581131:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1787764331.581131:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000100:00000040:2.0:1787764331.581151:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (22056)
00000800:00000200:3.0:1787764331.581160:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764331.581165:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787764331.581171:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764331.581174:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787764331.581180:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764331.581183:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764331.581187:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764331.581191:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764331.581196:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764331.581199:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764331.581203:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764331.581207:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658780 x1874599832714112/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764356 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.581216:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764331.581220:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658780 x1874599832714112/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764356 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.581227:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764331.581228:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764331.581235:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764331.581238:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764331.581240:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316719860
00000100:00000010:3.0:1787764331.581246:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d408a0.
00000100:00000001:3.0:1787764331.581253:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787764331.581260:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316719860
00000100:00000010:3.0:1787764331.581262:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40c00.
00000100:00000001:3.0:1787764331.581265:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787764331.581268:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764331.581270:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764331.581272:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764331.581276:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764331/real 1787764331]  req@ffff9751f8658780 x1874599832714112/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764331.581284:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764331.581286:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764331.581288:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764331.581290:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764331.581292:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764331.581295:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764331.581298:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658780 x1874599832714112/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764331.581306:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658780 x1874599832714112/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.581313:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764331.581314:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.581317:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764331.581318:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764331.581322:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658780 x1874599832714112/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.581329:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764331.581334:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658780 x1874599832714112/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.581341:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764331.581344:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.581346:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764331.581347:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.581350:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764331.581352:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764331.581355:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764331.581359:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764331.581361:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764331.581365:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658780 x1874599832714112/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764331.581372:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832714112:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764331.581379:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764331.581382:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832714240/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.581389:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764331.581390:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764331.581392:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764331.581394:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764331.581396:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764331.581398:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832714240:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764331.581401:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764331.581403:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764331.581404:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764331.581406:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764331.581410:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764331.581417:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764331.581420:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764331.581423:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764331.581427:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764331.581430:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832714240, portal 10
00000100:00000001:3.0:1787764331.581432:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764331.581433:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:3.0:1787764331.581440:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832714240/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764356 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.581446:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764331.581448:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764331.581450:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764331.581452:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832714240, offset 0
00000400:00000200:3.0:1787764331.581457:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764331.581460:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764331.581464:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764331.581466:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764331.581470:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764331.582260:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764331.582266:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764331.582270:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58815:58815:256:4294967295] 192.168.203.110@tcp LPNI seq info [58815:58815:8:4294967295]
00000400:00000200:3.0:1787764331.582276:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764331.582283:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764331.582289:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764331.582309:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764331.582315:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764331.582319:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787764331.582324:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764331.582327:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787764331.582331:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764331.582333:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764331.582337:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764331.582341:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764331.582345:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764331.582347:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764331.582357:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764331.582366:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832714240/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764356 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.582375:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764331.582379:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832714240/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764356 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.582386:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764331.582388:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764331.582393:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764331.582397:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764331.582401:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764331.582404:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316718625
00000100:00000001:3.0:1787764331.582407:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316719860
00000100:00000001:3.0:1787764331.582409:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316718625
00000100:00000001:3.0:1787764331.582412:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316719860
00000100:00000001:3.0:1787764331.582414:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764331.582416:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764331.582419:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764331.582423:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764331/real 1787764331]  req@ffff9751f865ad00 x1874599832714240/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764331.582431:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764331.582433:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764331.582434:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764331.582444:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764331.582447:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764331.582452:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764331.582456:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832714240/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764331.582464:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832714240/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.582471:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764331.582472:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.582474:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764331.582476:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764331.582479:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832714240/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.582487:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764331.582491:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832714240/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.582498:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764331.582500:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.582502:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764331.582503:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.582506:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764331.582519:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764331.582521:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764331.582525:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764331.582527:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764331.582531:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832714240/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764331.582540:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832714240:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764331.582544:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.582546:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764331.582550:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658780 x1874599832714112/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.582556:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764331.582559:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764331.582562:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764331.582565:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764331.582567:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764331.582569:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764331.582571:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764331.582573:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f4c00.
02000000:00000001:3.0:1787764331.582577:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764331.582579:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764331.582581:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658780.
00000100:00000001:3.0:1787764331.582584:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764331.582585:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.582587:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764331.582590:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832714240/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.582596:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764331.582598:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764331.582599:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764331.582602:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764331.582603:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764331.582604:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764331.582606:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764331.582607:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f4000.
02000000:00000001:3.0:1787764331.582609:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764331.582611:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764331.582612:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:3.0:1787764331.582615:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764331.582616:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.582618:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.582625:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764331.582627:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764331.582629:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764331.582631:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764331.582632:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764331.582637:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832714368/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.582645:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764331.582646:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764331.582648:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764331.582650:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764331.582653:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764331.582656:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832714368:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764331.582659:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764331.582662:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764331.582664:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764331.582665:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764331.582669:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764331.582671:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764331.582674:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764331.582677:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764331.582680:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764331.582684:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832714368, portal 10
00000100:00000001:3.0:1787764331.582686:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764331.582688:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:3.0:1787764331.582693:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832714368/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764356 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.582700:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764331.582702:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764331.582704:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764331.582707:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832714368, offset 0
00000400:00000200:3.0:1787764331.582711:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764331.582715:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764331.582719:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764331.582723:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764331.582726:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764331.582728:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764331.582730:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764331.582733:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58816:58816:256:4294967295] 192.168.203.110@tcp LPNI seq info [58816:58816:8:4294967295]
00000400:00000200:3.0:1787764331.582738:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764331.582743:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764331.582747:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764331.582750:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764331.582754:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764331.582757:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787764331.582761:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764331.582764:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787764331.582768:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764331.582770:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764331.582773:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764331.582776:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764331.582779:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764331.582781:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764331.582783:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764331.582787:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832714368/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764356 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.582794:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764331.582798:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832714368/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764356 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.582805:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764331.582807:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764331.582810:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764331.582812:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764331.582816:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764331.582818:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316718613
00000100:00000001:3.0:1787764331.582820:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316719860
00000100:00000001:3.0:1787764331.582822:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316718613
00000100:00000001:3.0:1787764331.582824:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316719860
00000100:00000001:3.0:1787764331.582826:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764331.582828:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764331.582830:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764331.582834:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764331/real 1787764331]  req@ffff9751f86583c0 x1874599832714368/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764331.582841:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764331.582842:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764331.582844:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764331.582846:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764331.582848:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764331.582850:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764331.582853:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832714368/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764331.582862:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832714368/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.582869:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764331.582870:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.582872:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764331.582873:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764331.582884:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832714368/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.582894:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764331.582898:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832714368/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.582905:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764331.582907:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.582909:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764331.582910:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.582912:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764331.582913:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764331.582915:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764331.582918:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764331.582921:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764331.582924:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832714368/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764331.582932:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832714368:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764331.582949:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764331.582953:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832714496/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.582959:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764331.582960:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764331.582962:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764331.582964:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764331.582965:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764331.582968:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832714496:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764331.582971:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764331.582973:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764331.582975:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764331.582977:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764331.582980:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764331.582982:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764331.582985:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764331.582987:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764331.582990:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764331.582994:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832714496, portal 4
00000100:00000001:3.0:1787764331.582995:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764331.582997:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:3.0:1787764331.583002:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832714496/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764356 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.583009:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764331.583012:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764331.583014:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764331.583016:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832714496, offset 0
00000400:00000200:3.0:1787764331.583020:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764331.583024:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764331.583027:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764331.583030:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764331.583032:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764331.583035:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764331.583037:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764331.583040:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58817:58817:256:4294967295] 192.168.203.110@tcp LPNI seq info [58817:58817:8:4294967295]
00000400:00000200:3.0:1787764331.583044:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764331.583050:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764331.583055:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764331.583058:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764331.583061:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764331.583064:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000200:3.1:1787764331.583068:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764331.583071:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 22 secs later)
00000800:00000100:3.0:1787764331.583075:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764331.583077:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764331.583080:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764331.583083:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764331.583086:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764331.583087:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764331.583089:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764331.583093:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832714496/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764356 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.583101:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764331.583104:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832714496/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764356 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.583111:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764331.583112:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764331.583115:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764331.583117:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764331.583120:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764331.583122:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316718630
00000100:00000001:3.0:1787764331.583125:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316719860
00000100:00000001:3.0:1787764331.583127:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316718630
00000100:00000001:3.0:1787764331.583129:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316719860
00000100:00000001:3.0:1787764331.583131:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764331.583133:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764331.583152:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764331.583156:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764331/real 1787764331]  req@ffff9751f8659a40 x1874599832714496/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764331.583164:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764331.583166:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764331.583168:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764331.583170:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764331.583172:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764331.583174:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764331.583177:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832714496/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764331.583186:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832714496/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.583192:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764331.583194:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.583195:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764331.583196:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764331.583200:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832714496/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.583206:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764331.583209:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832714496/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.583215:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764331.583216:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.583217:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764331.583218:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.583219:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764331.583220:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764331.583221:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764331.583223:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764331.583224:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764331.583226:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832714496/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764331.583231:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832714496:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764331.583233:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.583235:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764331.583237:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832714368/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.583241:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764331.583242:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764331.583243:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764331.583244:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764331.583245:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764331.583246:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764331.583247:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764331.583248:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f5000.
02000000:00000001:3.0:1787764331.583250:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764331.583251:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764331.583252:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:3.0:1787764331.583254:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764331.583254:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.583256:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764331.583257:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832714496/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764356 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764331.583261:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764331.583261:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764331.583262:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764331.583263:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764331.583264:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764331.583264:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764331.583265:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764331.583266:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f6800.
02000000:00000001:3.0:1787764331.583267:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764331.583267:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764331.583268:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:3.0:1787764331.583269:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764331.583270:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.583271:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764331.583273:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764331.583274:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764331.583275:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764331.583277:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764332.220225:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764332.220229:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764332.220232:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764332.220233:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764332.220236:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764332.220239:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764332.220239:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764332.220241:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764332.220243:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764332.220246:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764332.220247:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764332.220248:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764332.220250:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764332.220252:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764332.220254:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764332.220306:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764332.220309:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764332.220313:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764332.604265:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764332.604273:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764332.604275:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764332.604281:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764332.604285:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764332.604288:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764332.604290:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764332.604293:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764332.604295:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764332.860192:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316719860, at 4316719872
00000100:00000010:3.0:1787764332.860208:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40c00.
00000100:00000040:2.0:1787764332.860223:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316719860, at 4316719872
00000100:00000010:2.0:1787764332.860229:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d408a0.
00000100:00000001:2.0:1787764333.244225:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764333.244226:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764333.244230:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764333.244231:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764333.244234:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764333.244236:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764333.244237:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764333.244238:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764333.244242:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764333.244244:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764333.244246:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764333.244247:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764333.244249:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764333.244251:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764333.244254:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764333.244303:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764333.244307:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764333.244310:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764333.628230:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764333.628237:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764333.628240:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764333.628245:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764333.628250:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764333.628254:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764333.628256:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764333.628258:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764333.628261:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764334.268210:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764334.268216:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764334.268218:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764334.268222:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764334.268225:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764334.268227:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764334.268229:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764334.268231:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764334.268232:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764334.268305:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764334.268307:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764334.268309:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764334.268310:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764334.268312:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764334.268315:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764334.268317:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764334.268319:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764334.268321:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764334.268323:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764334.268324:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764334.268325:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764334.268377:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764334.268381:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764334.268384:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764334.652230:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764334.652238:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764334.652240:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764334.652245:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764334.652249:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764334.652252:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764334.652254:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764334.652257:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764334.652260:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764335.292224:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764335.292229:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764335.292232:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764335.292237:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764335.292239:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764335.292245:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00100000:2.0:1787764335.292247:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764335.292251:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764335.292253:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764335.292259:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764335.292260:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764335.292264:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764335.292265:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764335.292269:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764335.292271:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764335.292275:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764335.292277:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764335.292285:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764335.292287:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764335.292294:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764335.292296:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764335.292326:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764335.292330:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764335.292333:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764335.676237:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764335.676248:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764335.676251:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764335.676258:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764335.676262:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764335.676267:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764335.676269:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764335.676272:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764335.676276:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764336.316220:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764336.316227:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764336.316232:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764336.316234:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764336.316238:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764336.316240:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764336.316242:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764336.316251:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764336.316254:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764336.316260:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764336.316264:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764336.316268:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764336.316271:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764336.316273:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764336.316277:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764336.316299:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764336.316304:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764336.316308:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764336.700234:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764336.700243:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764336.700246:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764336.700251:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:2.0:1787764336.700257:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764336.700257:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764336.700262:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1787764336.700274:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764336.700274:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764336.700277:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1787764336.700279:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:2.0:1787764336.700281:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764336.700286:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764336.700290:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764336.700300:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 22051
00000400:00000200:2.0:1787764336.700310:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764336.700320:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764336.700324:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764336.700328:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764336.700337:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764336.700339:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764336.700342:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764336.700347:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764336.700350:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:2.0:1787764336.700353:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764336.700357:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:2.0:1787764336.700359:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:2.0:1787764336.700362:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:2.0:1787764336.700365:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764336.700366:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764336.700368:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764336.700370:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764336.700372:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:2.0:1787764336.700375:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764336.700376:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764336.700379:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:2.0:1787764336.700382:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764336.700384:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764336.700386:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:2.0:1787764336.700388:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:2.0:1787764336.700398:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8658000.
00000020:00000040:2.0:1787764336.700401:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:2.0:1787764336.700405:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764336.700406:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764336.700409:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764336.700412:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764336.700415:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764336.700422:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f7c00.
00000100:00000001:2.0:1787764336.700428:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764336.700437:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8658000 x1874599832714624/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764336.700455:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764336.700461:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764336.700466:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005806b0e6] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8658000 x1874599832714624/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764336.700492:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764336.700500:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764336.700503:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:2.0:1787764336.700504:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764336.700505:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764336.700507:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764336.700509:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22051
00000400:00000200:2.0:1787764336.700512:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764336.700515:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764336.700517:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764336.700520:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764336.700521:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764336.700525:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764336.700527:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764336.700529:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764336.700529:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:2.0:1787764336.700531:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787764336.700532:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000020:00000040:2.0:1787764336.700533:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:2.0:1787764336.700538:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:2.0:1787764336.700541:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000001:3.0:1787764336.700542:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:2.0:1787764336.700547:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764336.700549:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:2.0:1787764336.700550:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764336.700555:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764336.700556:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764336.700558:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:2.0:1787764336.700563:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764336.700567:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764336.700570:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000040:3.0:1787764336.700570:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8658000 x1874599832714624/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:2.0:1787764336.700575:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764336.700579:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764336.700587:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764336.700595:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787764336.700597:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000008:00000001:2.0:1787764336.700602:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764336.700602:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764336.700609:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764336.700611:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764336.700616:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000010:2.0:1787764336.700618:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f8659a40.
00000020:00000040:2.0:1787764336.700622:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787764336.700622:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764336.700631:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764336.700633:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764336.700634:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764336.700635:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764336.700637:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764336.700640:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f4800.
00000100:00100000:3.0:1787764336.700642:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832714624:192.168.203.110@tcp:250:kworker.0
00000100:00000001:2.0:1787764336.700645:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764336.700652:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00100000:2.0:1787764336.700654:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f8659a40 x1874599832714752/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
02000000:00000001:3.0:1787764336.700658:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764336.700663:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764336.700664:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764336.700671:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764336.700674:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000040:2.0:1787764336.700680:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000ba72929] to pc [ptlrpcd_rcv+-1]  req@ffff9751f8659a40 x1874599832714752/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1787764336.700687:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
00000100:00000001:2.0:1787764336.700690:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764336.700692:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:2.0:1787764336.700695:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764336.700700:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:2.0:1787764336.700702:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:2.0:1787764336.700704:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000400:00000200:3.0:1787764336.700708:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:2.0:1787764336.700709:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764336.700717:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22051
00000400:00000010:3.0:1787764336.700718:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000200:2.0:1787764336.700723:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:2.0:1787764336.700727:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764336.700728:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000010:3.0:1787764336.700730:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000040:2.0:1787764336.700731:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764336.700738:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764336.700741:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764336.700742:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000200:3.0:1787764336.700743:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832714624, portal 25
00000020:00000001:2.0:1787764336.700745:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764336.700748:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000100:00000001:3.0:1787764336.700750:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000020:00000001:2.0:1787764336.700751:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:2.0:1787764336.700756:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000100:00000001:3.0:1787764336.700757:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048387584 : -115096661164032 : ffff9751f8658000)
00000020:00000001:2.0:1787764336.700760:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:2.0:1787764336.700768:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:2.0:1787764336.700776:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000040:3.0:1787764336.700778:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8658000 x1874599832714624/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764361 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764336.700779:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764336.700785:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764336.700792:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764336.700801:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:2.0:1787764336.700808:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:2.0:1787764336.700814:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764336.700817:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764336.700818:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:2.0:1787764336.700819:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764336.700823:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764336.700824:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764336.700827:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000040:3.0:1787764336.700830:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000008:00000001:2.0:1787764336.700831:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764336.700834:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764336.700836:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000010:2.0:1787764336.700837:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f86583c0.
00000020:00000040:2.0:1787764336.700842:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:2.0:1787764336.700845:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000200:3.0:1787764336.700845:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832714624, offset 0
02000000:00000001:2.0:1787764336.700848:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764336.700852:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764336.700854:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:2.0:1787764336.700855:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:2.0:1787764336.700858:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f7400.
00000100:00000001:2.0:1787764336.700860:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764336.700866:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00100000:2.0:1787764336.700870:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f86583c0 x1874599832714880/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:2.0:1787764336.700876:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764336.700878:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764336.700884:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:2.0:1787764336.700885:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5946c08] to pc [ptlrpcd_rcv+-1]  req@ffff9751f86583c0 x1874599832714880/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764336.700898:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764336.700911:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:2.0:1787764336.700916:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:3.0:1787764336.700916:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00080000:2.0:1787764336.700922:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:2.0:1787764336.700926:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:2.0:1787764336.700928:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1787764336.700928:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00080000:2.0:1787764336.700930:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:2.0:1787764336.700933:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:2.0:1787764336.700939:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22051
00000400:00000200:3.0:1787764336.700940:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764336.700947:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764336.700950:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764336.700954:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58818:58818:256:4294967295] 192.168.203.110@tcp LPNI seq info [58818:58818:8:4294967295]
00000400:00000200:3.0:1787764336.700959:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764336.700967:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000400:00000200:2.0:1787764336.700976:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:3.0:1787764336.700981:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:2.0:1787764336.700984:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:2.0:1787764336.700988:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:2.0:1787764336.700992:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764336.700994:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000010:3.0:1787764336.700995:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000100:00000040:2.0:1787764336.700996:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764336.701002:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:2.0:1787764336.701006:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:2.0:1787764336.701010:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:2.0:1787764336.701013:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000200:3.0:1787764336.701014:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000040:2.0:1787764336.701015:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:2.0:1787764336.701021:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:2.0:1787764336.701023:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000800:00000200:3.0:1787764336.701025:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000100:00000040:2.0:1787764336.701028:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764336.701032:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:2.0:1787764336.701035:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:2.0:1787764336.701037:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:2.0:1787764336.701039:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:2.0:1787764336.701046:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000800:00000200:3.1:1787764336.701046:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:2.0:1787764336.701054:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:2.0:1787764336.701058:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000200:3.1:1787764336.701059:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
02000000:00000001:2.0:1787764336.701062:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:2.0:1787764336.701069:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764336.701071:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:2.0:1787764336.701074:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000800:00000100:3.0:1787764336.701077:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000008:00000001:2.0:1787764336.701078:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764336.701083:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1787764336.701085:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000100:00000010:2.0:1787764336.701089:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f865ad00.
00000400:00000200:3.0:1787764336.701093:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:2.0:1787764336.701094:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:2.0:1787764336.701102:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:2.0:1787764336.701106:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764336.701107:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:2.0:1787764336.701114:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:2.0:1787764336.701117:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1787764336.701118:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
02000000:00000001:2.0:1787764336.701120:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764336.701123:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
02000000:00000010:2.0:1787764336.701128:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86f6400.
00000100:00000001:3.0:1787764336.701131:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:2.0:1787764336.701158:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764336.701165:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f865ad00 x1874599832715008/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000200:3.0:1787764336.701177:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8658000 x1874599832714624/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764361 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:2.0:1787764336.701181:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:2.0:1787764336.701185:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764336.701194:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005ca60ca7] to pc [ptlrpcd_rcv+-1]  req@ffff9751f865ad00 x1874599832715008/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.701207:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:2.0:1787764336.701209:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:2.0:1787764336.701218:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (22061)
00000100:00000040:3.0:1787764336.701220:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8658000 x1874599832714624/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764361 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.701232:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764336.701234:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764336.701241:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764336.701245:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764336.701248:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316724980
00000100:00000010:3.0:1787764336.701254:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40c00.
00000100:00000001:3.0:1787764336.701259:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:3.0:1787764336.701266:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316724980
00000100:00000010:3.0:1787764336.701269:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40e40.
00000100:00000001:3.0:1787764336.701272:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787764336.701274:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764336.701277:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764336.701279:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764336.701284:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764336/real 1787764336]  req@ffff9751f8658000 x1874599832714624/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764336.701292:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764336.701294:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764336.701296:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764336.701298:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764336.701300:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764336.701302:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764336.701306:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8658000 x1874599832714624/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764336.701314:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8658000 x1874599832714624/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.701319:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764336.701321:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764336.701323:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764336.701325:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764336.701328:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8658000 x1874599832714624/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.701336:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764336.701340:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8658000 x1874599832714624/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.701347:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764336.701350:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764336.701352:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764336.701353:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764336.701355:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764336.701357:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764336.701361:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764336.701364:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764336.701367:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764336.701371:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8658000 x1874599832714624/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764336.701379:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8658000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832714624:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764336.701383:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764336.701385:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764336.701388:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8658000 x1874599832714624/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.701394:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764336.701397:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764336.701399:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764336.701402:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764336.701405:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764336.701406:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764336.701408:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764336.701411:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f7c00.
02000000:00000001:3.0:1787764336.701414:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764336.701417:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764336.701419:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8658000.
00000100:00000001:3.0:1787764336.701422:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764336.701423:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764336.701425:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764336.701431:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764336.701433:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764336.701435:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764336.701436:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764336.701438:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764336.701442:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f8659a40 x1874599832714752/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.701449:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764336.701451:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764336.701453:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764336.701455:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764336.701457:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764336.701460:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832714752:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764336.701463:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764336.701465:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764336.701466:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764336.701468:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764336.701471:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764336.701473:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764336.701476:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764336.701479:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764336.701483:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764336.701485:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832714752, portal 10
00000100:00000001:3.0:1787764336.701487:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764336.701488:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048394304 : -115096661157312 : ffff9751f8659a40)
00000100:00000040:3.0:1787764336.701494:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f8659a40 x1874599832714752/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764361 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.701500:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764336.701502:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764336.701505:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764336.701507:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832714752, offset 0
00000400:00000200:3.0:1787764336.701511:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764336.701515:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764336.701518:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764336.701521:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764336.701523:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764336.701526:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764336.701528:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764336.701531:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58819:58819:256:4294967295] 192.168.203.110@tcp LPNI seq info [58819:58819:8:4294967295]
00000400:00000200:3.0:1787764336.701535:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764336.701541:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764336.701546:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764336.701549:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764336.701553:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764336.701556:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787764336.701560:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764336.701563:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787764336.701567:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764336.701569:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764336.701572:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764336.701575:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764336.701578:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764336.701580:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764336.701582:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764336.701586:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f8659a40 x1874599832714752/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764361 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.701593:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764336.701596:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f8659a40 x1874599832714752/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764361 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.701602:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764336.701604:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764336.701608:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764336.701611:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764336.701614:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764336.701616:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316723744
00000100:00000001:3.0:1787764336.701619:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316724980
00000100:00000001:3.0:1787764336.701621:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316723744
00000100:00000001:3.0:1787764336.701623:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316724980
00000100:00000001:3.0:1787764336.701624:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764336.701626:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764336.701628:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764336.701632:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764336/real 1787764336]  req@ffff9751f8659a40 x1874599832714752/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764336.701639:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764336.701641:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764336.701642:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764336.701645:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764336.701646:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764336.701648:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764336.701651:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f8659a40 x1874599832714752/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764336.701659:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f8659a40 x1874599832714752/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.701665:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764336.701667:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764336.701668:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764336.701670:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764336.701673:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f8659a40 x1874599832714752/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.701679:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764336.701683:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f8659a40 x1874599832714752/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.701690:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764336.701692:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764336.701693:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764336.701694:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764336.701696:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764336.701708:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764336.701711:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764336.701714:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764336.701716:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764336.701720:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f8659a40 x1874599832714752/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764336.701728:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f8659a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832714752:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764336.701732:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764336.701735:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f86583c0 x1874599832714880/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.701741:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764336.701743:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764336.701745:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764336.701747:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764336.701749:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764336.701751:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832714880:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764336.701754:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764336.701757:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764336.701758:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764336.701760:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764336.701764:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764336.701766:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764336.701769:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764336.701772:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764336.701775:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764336.701777:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832714880, portal 10
00000100:00000001:3.0:1787764336.701779:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764336.701780:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048388544 : -115096661163072 : ffff9751f86583c0)
00000100:00000040:3.0:1787764336.701786:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f86583c0 x1874599832714880/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764361 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.701792:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764336.701794:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764336.701796:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764336.701798:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832714880, offset 0
00000400:00000200:3.0:1787764336.701802:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764336.701806:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764336.701809:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764336.701812:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764336.701815:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764336.701817:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764336.701820:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764336.701823:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58820:58820:256:4294967295] 192.168.203.110@tcp LPNI seq info [58820:58820:8:4294967295]
00000400:00000200:3.0:1787764336.701827:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764336.701833:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764336.701837:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764336.701840:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764336.701844:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764336.701847:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787764336.701851:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764336.701854:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787764336.701858:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764336.701860:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764336.701863:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764336.701866:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764336.701869:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764336.701871:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764336.701874:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764336.701877:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f86583c0 x1874599832714880/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764361 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.701884:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764336.701887:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f86583c0 x1874599832714880/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764361 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.701894:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764336.701895:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764336.701898:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764336.701913:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764336.701917:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764336.701919:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316723732
00000100:00000001:3.0:1787764336.701921:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316724980
00000100:00000001:3.0:1787764336.701923:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316723732
00000100:00000001:3.0:1787764336.701925:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316724980
00000100:00000001:3.0:1787764336.701927:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764336.701928:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764336.701930:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764336.701934:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764336/real 1787764336]  req@ffff9751f86583c0 x1874599832714880/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764336.701942:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764336.701943:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764336.701944:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764336.701946:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764336.701948:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764336.701950:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764336.701953:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f86583c0 x1874599832714880/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764336.701961:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f86583c0 x1874599832714880/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.701968:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764336.701969:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764336.701970:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764336.701972:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764336.701975:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f86583c0 x1874599832714880/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.701981:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764336.701985:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f86583c0 x1874599832714880/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.701991:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764336.701993:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764336.701995:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764336.701996:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764336.701998:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764336.701999:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764336.702002:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764336.702005:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764336.702007:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764336.702011:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f86583c0 x1874599832714880/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764336.702018:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f86583c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832714880:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764336.702021:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764336.702025:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f865ad00 x1874599832715008/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.702031:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764336.702032:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764336.702034:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764336.702036:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764336.702038:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764336.702041:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832715008:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764336.702044:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764336.702046:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764336.702048:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764336.702049:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764336.702053:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764336.702055:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764336.702058:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764336.702060:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764336.702063:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764336.702067:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832715008, portal 4
00000100:00000001:3.0:1787764336.702069:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764336.702070:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977048399104 : -115096661152512 : ffff9751f865ad00)
00000100:00000040:3.0:1787764336.702075:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f865ad00 x1874599832715008/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764361 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.702080:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764336.702082:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764336.702084:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764336.702086:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832715008, offset 0
00000400:00000200:3.0:1787764336.702090:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764336.702094:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764336.702097:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764336.702100:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764336.702102:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764336.702104:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764336.702106:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764336.702109:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58821:58821:256:4294967295] 192.168.203.110@tcp LPNI seq info [58821:58821:8:4294967295]
00000400:00000200:3.0:1787764336.702113:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764336.702118:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764336.702123:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764336.702126:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764336.702130:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764336.702153:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000200:3.1:1787764336.702158:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764336.702161:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 17 secs later)
00000800:00000100:3.0:1787764336.702164:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764336.702166:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764336.702169:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764336.702172:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764336.702175:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764336.702177:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764336.702179:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764336.702182:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f865ad00 x1874599832715008/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764361 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.702190:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764336.702193:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f865ad00 x1874599832715008/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764361 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.702200:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764336.702201:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764336.702204:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764336.702207:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764336.702210:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764336.702212:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316723750
00000100:00000001:3.0:1787764336.702214:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316724980
00000100:00000001:3.0:1787764336.702216:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316723750
00000100:00000001:3.0:1787764336.702218:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316724980
00000100:00000001:3.0:1787764336.702220:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764336.702222:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764336.702224:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764336.702227:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764336/real 1787764336]  req@ffff9751f865ad00 x1874599832715008/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764336.702235:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764336.702236:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764336.702238:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764336.702241:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764336.702242:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764336.702244:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764336.702247:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f865ad00 x1874599832715008/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764336.702255:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f865ad00 x1874599832715008/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.702261:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764336.702262:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764336.702264:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764336.702265:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764336.702269:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f865ad00 x1874599832715008/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.702275:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764336.702278:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f865ad00 x1874599832715008/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.702285:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764336.702287:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764336.702288:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764336.702289:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764336.702291:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764336.702292:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764336.702294:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764336.702297:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764336.702299:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764336.702303:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f865ad00 x1874599832715008/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764336.702311:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f865ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832715008:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764336.702314:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764336.702315:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764336.702318:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f8659a40 x1874599832714752/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.702324:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764336.702325:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764336.702327:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764336.702329:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764336.702331:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764336.702332:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764336.702334:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764336.702335:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f4800.
02000000:00000001:3.0:1787764336.702337:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764336.702339:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764336.702340:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f8659a40.
00000100:00000001:3.0:1787764336.702342:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764336.702343:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764336.702344:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764336.702347:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f86583c0 x1874599832714880/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.702353:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764336.702354:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764336.702355:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764336.702358:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764336.702359:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764336.702360:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764336.702362:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764336.702363:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f7400.
02000000:00000001:3.0:1787764336.702365:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764336.702367:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764336.702368:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f86583c0.
00000100:00000001:3.0:1787764336.702370:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764336.702371:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764336.702372:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764336.702375:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f865ad00 x1874599832715008/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764361 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764336.702381:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764336.702382:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764336.702383:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764336.702385:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764336.702386:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764336.702387:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764336.702389:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764336.702391:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86f6400.
02000000:00000001:3.0:1787764336.702393:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764336.702394:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764336.702395:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f865ad00.
00000100:00000001:3.0:1787764336.702397:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764336.702398:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764336.702400:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764336.702403:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764336.702405:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764336.702406:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764336.702408:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764337.340248:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764337.340254:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764337.340260:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764337.340261:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764337.340263:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764337.340268:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764337.340271:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764337.340272:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764337.340277:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764337.340278:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764337.340281:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764337.340287:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764337.340289:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764337.340296:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764337.340297:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764337.340300:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764337.340301:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764337.340304:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764337.340310:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764337.340314:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764337.340318:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764337.340347:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764337.340351:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764337.340353:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764337.724239:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764337.724246:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764337.724248:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764337.724253:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764337.724257:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764337.724260:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764337.724263:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764337.724265:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764337.724267:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764337.980203:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316724980, at 4316724992
00000100:00000010:3.0:1787764337.980218:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40e40.
00000100:00000040:2.0:1787764337.980231:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316724980, at 4316724992
00000100:00000010:2.0:1787764337.980238:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40c00.
00000100:00000001:2.0:1787764338.364223:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764338.364223:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764338.364229:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764338.364231:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764338.364234:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764338.364235:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764338.364239:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764338.364240:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764338.364245:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764338.364246:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764338.364248:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764338.364250:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764338.364254:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764338.364256:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764338.364259:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764338.364288:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764338.364292:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764338.364295:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764338.748235:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764338.748243:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764338.748245:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764338.748249:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764338.748254:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764338.748257:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764338.748260:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764338.748262:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764338.748265:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764339.388234:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764339.388242:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764339.388245:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764339.388251:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764339.388255:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764339.388260:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764339.388262:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764339.388265:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764339.388267:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764339.388321:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764339.388325:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764339.388328:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764339.388329:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764339.388331:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764339.388332:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764339.388388:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764339.388392:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764339.388395:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764339.772215:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764339.772220:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764339.772222:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764339.772226:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764339.772229:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764339.772232:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764339.772234:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764339.772236:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764339.772237:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764340.412224:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764340.412225:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764340.412230:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764340.412231:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764340.412234:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764340.412236:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764340.412238:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764340.412239:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764340.412244:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764340.412245:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764340.412247:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764340.412249:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764340.412253:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764340.412255:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764340.412258:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764340.412288:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764340.412291:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764340.412293:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764340.796234:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764340.796243:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764340.796245:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764340.796251:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764340.796256:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764340.796258:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764340.796261:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764340.796263:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764340.796266:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764341.436223:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764341.436229:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764341.436229:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764341.436231:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764341.436234:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764341.436240:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764341.436241:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764341.436243:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764341.436245:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764341.436247:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764341.436248:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764341.436250:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764341.436252:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764341.436254:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764341.436256:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764341.436307:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764341.436311:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764341.436314:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764341.820205:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764341.820212:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764341.820213:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764341.820217:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764341.820219:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764341.820222:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.820225:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764341.820227:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.820230:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764341.820259:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764341.820265:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:1.0:1787764341.820268:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764341.820271:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764341.820275:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764341.820282:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 22056
00000400:00000200:1.0:1787764341.820289:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764341.820295:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764341.820298:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764341.820302:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764341.820304:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764341.820307:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764341.820309:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764341.820313:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764341.820315:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:1.0:1787764341.820317:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764341.820321:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:1.0:1787764341.820324:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:1.0:1787764341.820326:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:1.0:1787764341.820329:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764341.820331:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764341.820333:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764341.820335:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764341.820338:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:1.0:1787764341.820341:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764341.820344:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764341.820346:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764341.820348:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764341.820350:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764341.820352:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:1.0:1787764341.820354:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764341.820358:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7933480.
00000020:00000040:1.0:1787764341.820360:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:1.0:1787764341.820363:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764341.820364:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764341.820367:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764341.820369:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764341.820372:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764341.820377:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cb000.
00000100:00000001:1.0:1787764341.820382:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764341.820389:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7933480 x1874599832715136/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764341.820396:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764341.820399:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764341.820403:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a3c2c33c] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7933480 x1874599832715136/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764341.820420:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764341.820422:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764341.820425:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:1.0:1787764341.820426:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764341.820428:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764341.820430:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764341.820432:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22056
00000400:00000200:1.0:1787764341.820435:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764341.820439:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764341.820441:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764341.820445:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764341.820447:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764341.820450:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764341.820451:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764341.820454:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764341.820455:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:1.0:1787764341.820458:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1787764341.820459:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:1.0:1787764341.820462:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:1.0:1787764341.820466:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764341.820469:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:1.0:1787764341.820471:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:1.0:1787764341.820474:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:1.0:1787764341.820477:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764341.820480:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764341.820482:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764341.820485:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764341.820487:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:1.0:1787764341.820490:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764341.820493:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764341.820495:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764341.820499:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764341.820501:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764341.820503:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764341.820506:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:1.0:1787764341.820510:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764341.820512:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:1.0:1787764341.820516:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930000.
00000020:00000040:1.0:1787764341.820519:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:1.0:1787764341.820521:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764341.820523:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764341.820526:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764341.820527:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000040:3.0:1787764341.820527:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7933480 x1874599832715136/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787764341.820530:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764341.820535:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9c00.
00000100:00000001:1.0:1787764341.820540:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764341.820547:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930000 x1874599832715264/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764341.820555:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764341.820557:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764341.820560:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6d90274] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930000 x1874599832715264/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764341.820568:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:1.0:1787764341.820571:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:1.0:1787764341.820574:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:1.0:1787764341.820576:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764341.820577:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764341.820580:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:1.0:1787764341.820582:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22056
00000400:00000200:1.0:1787764341.820586:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764341.820589:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764341.820592:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764341.820594:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764341.820597:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764341.820599:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764341.820601:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:1.0:1787764341.820604:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:1.0:1787764341.820605:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:1.0:1787764341.820607:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764341.820609:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:1.0:1787764341.820611:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:1.0:1787764341.820613:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:1.0:1787764341.820616:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764341.820618:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764341.820620:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764341.820622:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:1.0:1787764341.820625:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:1.0:1787764341.820630:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764341.820635:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764341.820636:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
02000000:00000001:1.0:1787764341.820639:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764341.820639:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764341.820643:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764341.820644:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764341.820648:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764341.820648:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764341.820652:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764341.820653:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:1.0:1787764341.820657:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00100000:3.0:1787764341.820659:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832715136:192.168.203.110@tcp:250:kworker.0
00000008:00000001:1.0:1787764341.820661:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764341.820664:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:1.0:1787764341.820665:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764341.820669:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000010:1.0:1787764341.820670:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7930b40.
02000000:00000001:3.0:1787764341.820672:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:1.0:1787764341.820674:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:3.0:1787764341.820675:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:1.0:1787764341.820679:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:1.0:1787764341.820681:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764341.820681:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
00000100:00000001:1.0:1787764341.820686:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:1.0:1787764341.820687:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:1.0:1787764341.820689:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764341.820693:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86c9400.
00000100:00000001:1.0:1787764341.820697:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764341.820703:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7930b40 x1874599832715392/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764341.820712:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764341.820713:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764341.820716:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022d7f91a] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7930b40 x1874599832715392/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764341.820723:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764341.820726:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
02000000:00000001:3.0:1787764341.820726:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764341.820732:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000400:00000200:3.0:1787764341.820734:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:1.0:1787764341.820737:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000010:3.0:1787764341.820738:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000100:00080000:1.0:1787764341.820744:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000400:00000010:3.0:1787764341.820745:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000001:1.0:1787764341.820746:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:1.0:1787764341.820750:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000200:3.0:1787764341.820753:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832715136, portal 25
00000100:00000001:1.0:1787764341.820754:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764341.820759:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00080000:1.0:1787764341.820761:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22056
00000100:00000001:3.0:1787764341.820762:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497734784 : -115097211816832 : ffff9751d7933480)
00000400:00000200:1.0:1787764341.820768:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:3.0:1787764341.820772:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7933480 x1874599832715136/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764366 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1787764341.820775:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:1.0:1787764341.820780:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:1.0:1787764341.820786:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764341.820790:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:1.0:1787764341.820791:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:1.0:1787764341.820797:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000040:3.0:1787764341.820797:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787764341.820802:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000010:3.0:1787764341.820802:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000020:00000001:1.0:1787764341.820807:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000200:3.0:1787764341.820808:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832715136, offset 0
00000020:00000040:1.0:1787764341.820810:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:1.0:1787764341.820813:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:1.0:1787764341.820815:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000400:00000200:3.0:1787764341.820816:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000020:00000001:1.0:1787764341.820820:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:1.0:1787764341.820823:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000200:3.0:1787764341.820826:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:1.0:1787764341.820830:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764341.820834:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787764341.820835:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000040:1.0:1787764341.820840:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000200:3.0:1787764341.820842:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000100:00000001:1.0:1787764341.820846:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:3.0:1787764341.820848:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000020:00000040:1.0:1787764341.820851:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000400:00000200:3.0:1787764341.820853:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00080000:1.0:1787764341.820859:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000200:3.0:1787764341.820859:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764341.820877:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58822:58822:256:4294967295] 192.168.203.110@tcp LPNI seq info [58822:58822:8:4294967295]
00000400:00000200:3.0:1787764341.820882:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764341.820890:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764341.820896:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:1.0:1787764341.820899:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764341.820903:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000800:00000010:3.0:1787764341.820903:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
02000000:00000001:1.0:1787764341.820907:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764341.820912:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787764341.820913:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:1.0:1787764341.820917:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000800:00000200:3.0:1787764341.820918:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000008:00100000:1.0:1787764341.820921:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000800:00000200:3.1:1787764341.820924:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000008:00000001:1.0:1787764341.820928:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.1:1787764341.820929:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000100:00000001:1.0:1787764341.820932:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000800:00000100:3.0:1787764341.820935:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764341.820938:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000100:00000010:1.0:1787764341.820939:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751d7931680.
00000400:00000200:3.0:1787764341.820941:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000040:1.0:1787764341.820945:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000400:00000200:3.0:1787764341.820946:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:1.0:1787764341.820951:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:3.0:1787764341.820952:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
02000000:00000001:1.0:1787764341.820954:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764341.820955:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:1.0:1787764341.820958:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:3.0:1787764341.820960:0:257575:0:(events.c:34:request_out_callback()) Process entered
02000000:00000001:1.0:1787764341.820962:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:3.0:1787764341.820964:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7933480 x1874599832715136/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764366 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:1.0:1787764341.820966:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764341.820974:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9751f86cac00.
00000100:00000001:3.0:1787764341.820974:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764341.820979:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7933480 x1874599832715136/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764366 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764341.820980:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764341.820988:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764341.820990:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00100000:1.0:1787764341.820991:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751d7931680 x1874599832715520/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764341.820995:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764341.821002:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764341.821006:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764341.821006:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316730099
00000100:00000001:1.0:1787764341.821011:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764341.821012:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40e40.
00000100:00000040:1.0:1787764341.821018:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d4255d6] to pc [ptlrpcd_rcv+-1]  req@ffff9751d7931680 x1874599832715520/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821020:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000001:1.0:1787764341.821029:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764341.821029:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316730099
00000100:00000010:3.0:1787764341.821036:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40000.
00000100:00000040:1.0:1787764341.821037:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (22066)
00000100:00000001:3.0:1787764341.821042:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:3.0:1787764341.821047:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764341.821050:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764341.821052:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764341.821058:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764341/real 1787764341]  req@ffff9751d7933480 x1874599832715136/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764341.821064:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764341.821065:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764341.821066:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764341.821068:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764341.821070:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764341.821071:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764341.821073:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7933480 x1874599832715136/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764341.821078:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7933480 x1874599832715136/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821082:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764341.821083:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.821084:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764341.821085:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764341.821088:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7933480 x1874599832715136/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821092:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764341.821094:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7933480 x1874599832715136/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821099:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764341.821100:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.821101:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764341.821102:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.821104:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764341.821105:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764341.821106:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764341.821109:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764341.821110:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764341.821113:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7933480 x1874599832715136/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764341.821118:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7933480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832715136:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764341.821121:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.821122:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764341.821124:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7933480 x1874599832715136/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821127:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764341.821129:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764341.821130:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764341.821132:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764341.821133:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764341.821150:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764341.821152:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764341.821154:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cb000.
02000000:00000001:3.0:1787764341.821156:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764341.821158:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764341.821159:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7933480.
00000100:00000001:3.0:1787764341.821162:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764341.821163:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.821164:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.821169:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764341.821170:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764341.821172:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764341.821173:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764341.821174:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764341.821177:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930000 x1874599832715264/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821182:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764341.821183:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764341.821184:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764341.821185:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764341.821188:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764341.821190:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832715264:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764341.821192:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764341.821194:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764341.821195:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764341.821197:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764341.821199:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764341.821200:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764341.821203:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764341.821204:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764341.821206:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764341.821208:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832715264, portal 10
00000100:00000001:3.0:1787764341.821209:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764341.821210:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497721344 : -115097211830272 : ffff9751d7930000)
00000100:00000040:3.0:1787764341.821214:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930000 x1874599832715264/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764366 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821218:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764341.821219:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764341.821220:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764341.821222:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832715264, offset 0
00000400:00000200:3.0:1787764341.821224:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764341.821226:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764341.821228:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764341.821230:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764341.821232:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764341.821233:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764341.821235:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764341.821237:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58823:58823:256:4294967295] 192.168.203.110@tcp LPNI seq info [58823:58823:8:4294967295]
00000400:00000200:3.0:1787764341.821239:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764341.821243:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764341.821245:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764341.821248:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764341.821250:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764341.821252:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787764341.821255:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764341.821256:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1787764341.821258:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764341.821259:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764341.821261:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764341.821263:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764341.821264:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764341.821266:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764341.821267:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764341.821269:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930000 x1874599832715264/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764366 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821274:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764341.821275:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930000 x1874599832715264/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764366 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821279:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764341.821280:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764341.821282:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764341.821284:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764341.821286:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764341.821287:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316728864
00000100:00000001:3.0:1787764341.821289:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316730099
00000100:00000001:3.0:1787764341.821290:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316728864
00000100:00000001:3.0:1787764341.821291:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316730099
00000100:00000001:3.0:1787764341.821293:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764341.821294:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764341.821295:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764341.821297:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764341/real 1787764341]  req@ffff9751d7930000 x1874599832715264/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764341.821305:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764341.821306:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764341.821307:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764341.821308:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764341.821309:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764341.821310:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764341.821311:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930000 x1874599832715264/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764341.821316:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930000 x1874599832715264/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821321:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764341.821322:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.821324:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764341.821325:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764341.821329:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930000 x1874599832715264/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821335:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764341.821338:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930000 x1874599832715264/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821345:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764341.821347:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.821349:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764341.821350:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.821352:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764341.821363:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764341.821365:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764341.821368:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764341.821370:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764341.821374:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930000 x1874599832715264/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764341.821382:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832715264:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764341.821386:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764341.821389:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7930b40 x1874599832715392/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821395:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764341.821397:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764341.821399:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764341.821401:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764341.821402:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764341.821405:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832715392:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764341.821408:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764341.821409:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764341.821410:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764341.821411:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764341.821414:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764341.821415:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764341.821417:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764341.821419:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764341.821421:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764341.821423:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832715392, portal 10
00000100:00000001:3.0:1787764341.821424:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764341.821425:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497724224 : -115097211827392 : ffff9751d7930b40)
00000100:00000040:3.0:1787764341.821429:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7930b40 x1874599832715392/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764366 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821432:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764341.821434:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764341.821435:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764341.821436:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832715392, offset 0
00000400:00000200:3.0:1787764341.821439:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764341.821441:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764341.821443:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764341.821445:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764341.821447:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764341.821448:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764341.821450:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764341.821451:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58824:58824:256:4294967295] 192.168.203.110@tcp LPNI seq info [58824:58824:8:4294967295]
00000400:00000200:3.0:1787764341.821454:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764341.821457:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764341.821460:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764341.821462:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764341.821464:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764341.821466:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787764341.821468:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764341.821470:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1787764341.821472:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764341.821473:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764341.821475:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764341.821477:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764341.821479:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764341.821480:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764341.821481:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764341.821483:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7930b40 x1874599832715392/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764366 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821487:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764341.821490:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7930b40 x1874599832715392/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764366 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821496:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764341.821498:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764341.821501:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764341.821503:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764341.821506:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764341.821508:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316728852
00000100:00000001:3.0:1787764341.821510:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316730099
00000100:00000001:3.0:1787764341.821512:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316728852
00000100:00000001:3.0:1787764341.821514:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316730099
00000100:00000001:3.0:1787764341.821516:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764341.821518:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764341.821520:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764341.821524:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764341/real 1787764341]  req@ffff9751d7930b40 x1874599832715392/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764341.821531:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764341.821532:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764341.821534:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764341.821535:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764341.821536:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764341.821537:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764341.821539:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7930b40 x1874599832715392/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764341.821544:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7930b40 x1874599832715392/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821548:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764341.821548:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.821550:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764341.821550:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764341.821552:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7930b40 x1874599832715392/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821556:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764341.821558:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7930b40 x1874599832715392/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821562:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764341.821563:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.821564:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764341.821565:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.821566:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764341.821567:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764341.821569:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764341.821570:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764341.821571:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764341.821573:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7930b40 x1874599832715392/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764341.821577:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7930b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832715392:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764341.821580:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764341.821581:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751d7931680 x1874599832715520/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821585:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764341.821586:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764341.821587:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764341.821588:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764341.821589:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764341.821591:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832715520:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764341.821593:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764341.821594:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764341.821595:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764341.821596:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764341.821598:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764341.821599:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764341.821601:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764341.821603:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764341.821605:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764341.821607:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832715520, portal 4
00000100:00000001:3.0:1787764341.821608:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764341.821609:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628976497727104 : -115097211824512 : ffff9751d7931680)
00000100:00000040:3.0:1787764341.821612:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751d7931680 x1874599832715520/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764366 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821616:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764341.821618:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764341.821619:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764341.821620:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832715520, offset 0
00000400:00000200:3.0:1787764341.821622:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764341.821624:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764341.821626:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764341.821628:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764341.821629:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764341.821631:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764341.821632:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764341.821634:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58825:58825:256:4294967295] 192.168.203.110@tcp LPNI seq info [58825:58825:8:4294967295]
00000400:00000200:3.0:1787764341.821637:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764341.821640:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764341.821643:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764341.821644:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764341.821646:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764341.821648:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000200:3.1:1787764341.821650:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764341.821652:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 12 secs later)
00000800:00000100:3.0:1787764341.821654:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764341.821655:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764341.821657:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764341.821658:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764341.821660:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764341.821661:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764341.821662:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764341.821664:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751d7931680 x1874599832715520/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764366 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821672:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764341.821674:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751d7931680 x1874599832715520/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764366 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821678:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764341.821678:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764341.821680:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764341.821682:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764341.821683:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764341.821685:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316728869
00000100:00000001:3.0:1787764341.821686:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316730099
00000100:00000001:3.0:1787764341.821687:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316728869
00000100:00000001:3.0:1787764341.821689:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316730099
00000100:00000001:3.0:1787764341.821690:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764341.821691:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764341.821692:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764341.821694:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764341/real 1787764341]  req@ffff9751d7931680 x1874599832715520/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764341.821698:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764341.821698:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764341.821699:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764341.821700:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764341.821701:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764341.821702:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764341.821704:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751d7931680 x1874599832715520/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764341.821708:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751d7931680 x1874599832715520/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821712:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764341.821713:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.821714:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764341.821715:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764341.821717:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751d7931680 x1874599832715520/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821720:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764341.821722:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751d7931680 x1874599832715520/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821726:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764341.821727:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.821728:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764341.821728:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.821729:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764341.821730:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764341.821731:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764341.821733:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764341.821734:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764341.821736:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751d7931680 x1874599832715520/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764341.821740:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751d7931680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832715520:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764341.821742:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.821743:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764341.821744:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930000 x1874599832715264/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821748:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764341.821749:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764341.821750:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764341.821752:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764341.821753:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764341.821755:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764341.821757:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764341.821758:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:3.0:1787764341.821761:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764341.821762:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764341.821763:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930000.
00000100:00000001:3.0:1787764341.821766:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764341.821767:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.821769:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764341.821772:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7930b40 x1874599832715392/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821778:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764341.821779:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764341.821780:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764341.821783:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764341.821784:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764341.821785:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764341.821787:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764341.821788:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86c9400.
02000000:00000001:3.0:1787764341.821790:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764341.821791:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764341.821792:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7930b40.
00000100:00000001:3.0:1787764341.821794:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764341.821796:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.821798:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764341.821800:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751d7931680 x1874599832715520/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764366 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764341.821804:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764341.821805:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764341.821805:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764341.821807:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764341.821807:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764341.821808:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764341.821809:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764341.821810:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9751f86cac00.
02000000:00000001:3.0:1787764341.821811:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764341.821812:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764341.821813:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751d7931680.
00000100:00000001:3.0:1787764341.821814:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764341.821815:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.821816:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764341.821819:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764341.821820:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764341.821821:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764341.821822:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764342.460209:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764342.460214:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764342.460217:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764342.460219:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764342.460220:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764342.460226:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764342.460227:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764342.460228:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764342.460233:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764342.460234:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764342.460237:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764342.460238:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764342.460242:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764342.460245:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764342.460247:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764342.460280:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764342.460283:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764342.460286:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764342.844231:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764342.844237:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764342.844239:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764342.844243:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764342.844246:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764342.844249:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764342.844251:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764342.844253:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764342.844255:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764343.100246:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316730099, at 4316730112
00000100:00000010:3.0:1787764343.100265:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40000.
00000100:00000040:2.0:1787764343.100280:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316730099, at 4316730112
00000100:00000010:2.0:1787764343.100293:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40e40.
00010000:00000001:3.0:1787764343.484243:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764343.484250:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764343.484256:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764343.484257:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764343.484260:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764343.484263:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764343.484265:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764343.484268:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764343.484271:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764343.484273:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764343.484276:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764343.484278:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764343.484296:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764343.484305:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764343.484308:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764343.484314:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764343.484319:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764343.484323:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764343.484326:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764343.484329:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764343.484333:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764343.484338:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764343.484343:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764343.484347:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764343.868213:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764343.868221:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764343.868223:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764343.868228:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764343.868231:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764343.868235:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764343.868237:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764343.868239:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764343.868241:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764344.508221:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764344.508223:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764344.508228:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764344.508229:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764344.508233:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764344.508236:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764344.508237:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764344.508239:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764344.508242:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764344.508243:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764344.508247:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764344.508250:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764344.508253:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764344.508255:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764344.508258:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764344.508347:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764344.508352:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764344.508355:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764344.892253:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764344.892263:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764344.892266:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764344.892273:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764344.892278:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764344.892282:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764344.892286:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764344.892288:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764344.892292:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764345.532255:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764345.532256:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764345.532266:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764345.532266:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764345.532268:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764345.532271:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764345.532272:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764345.532279:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00100000:2.0:1787764345.532280:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764345.532281:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764345.532284:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764345.532287:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764345.532287:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764345.532292:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764345.532293:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764345.532294:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764345.532300:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764345.532300:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764345.532303:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764345.532304:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764345.532309:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764345.532337:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764345.532342:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764345.532345:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764345.916221:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764345.916241:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764345.916245:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764345.916251:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764345.916256:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764345.916260:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764345.916263:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764345.916266:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764345.916269:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764346.556242:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764346.556245:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764346.556250:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764346.556250:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764346.556253:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764346.556256:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764346.556258:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764346.556259:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764346.556264:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764346.556265:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764346.556268:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764346.556268:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764346.556271:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764346.556273:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764346.556276:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764346.556300:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764346.556304:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764346.556307:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787764346.940223:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764346.940223:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764346.940231:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00080000:0.0:1787764346.940234:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764346.940236:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764346.940239:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:3.0:1787764346.940243:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:0.0:1787764346.940244:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764346.940249:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764346.940251:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:3.0:1787764346.940255:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.940260:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00080000:0.0:1787764346.940261:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 22061
00000100:00000001:3.0:1787764346.940265:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.940270:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787764346.940271:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764346.940280:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764346.940283:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764346.940288:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764346.940291:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764346.940294:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764346.940296:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764346.940300:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764346.940302:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764346.940305:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764346.940309:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764346.940312:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764346.940314:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787764346.940317:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764346.940319:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764346.940320:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764346.940322:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764346.940326:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787764346.940329:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764346.940331:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764346.940335:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764346.940339:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764346.940341:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764346.940344:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764346.940347:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764346.940354:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876de00.
00000020:00000040:0.0:1787764346.940357:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764346.940361:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764346.940363:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764346.940367:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764346.940369:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764346.940373:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764346.940381:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014800.
00000100:00000001:0.0:1787764346.940388:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764346.940397:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876de00 x1874599832715648/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764346.940408:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764346.940412:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764346.940416:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d458d2] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876de00 x1874599832715648/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764346.940435:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764346.940438:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764346.940441:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764346.940442:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764346.940444:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764346.940447:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764346.940450:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22061
00000400:00000200:0.0:1787764346.940453:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764346.940456:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764346.940459:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764346.940462:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764346.940464:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764346.940465:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1787764346.940468:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764346.940471:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:0.0:1787764346.940472:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764346.940476:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1787764346.940476:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1787764346.940479:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787764346.940481:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764346.940484:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000040:3.0:1787764346.940486:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876de00 x1874599832715648/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787764346.940487:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787764346.940491:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787764346.940498:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764346.940503:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764346.940505:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000040:0.0:1787764346.940507:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764346.940508:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764346.940510:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764346.940514:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
02000000:00000001:3.0:1787764346.940514:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00080000:0.0:1787764346.940517:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:3.0:1787764346.940520:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000001:0.0:1787764346.940521:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764346.940526:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764346.940527:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764346.940532:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00100000:3.0:1787764346.940534:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832715648:192.168.203.110@tcp:250:kworker.0
02000000:00000001:0.0:1787764346.940536:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764346.940541:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764346.940542:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000008:00100000:0.0:1787764346.940546:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
02000000:00000001:3.0:1787764346.940549:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000008:00000001:0.0:1787764346.940550:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764346.940554:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764346.940554:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764346.940560:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876da40.
02000000:00000001:3.0:1787764346.940561:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:0.0:1787764346.940565:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787764346.940568:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764346.940570:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764346.940570:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
00000100:00000001:0.0:1787764346.940575:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:3.0:1787764346.940575:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764346.940578:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764346.940581:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764346.940585:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000010:0.0:1787764346.940586:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:0.0:1787764346.940590:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764346.940591:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000100:00100000:0.0:1787764346.940597:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876da40 x1874599832715776/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000010:3.0:1787764346.940601:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000001:0.0:1787764346.940608:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764346.940610:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764346.940611:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832715648, portal 25
00000100:00000040:0.0:1787764346.940615:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049d94e1f] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876da40 x1874599832715776/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.940617:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764346.940620:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049525760 : -115096660025856 : ffff9751f876de00)
00000100:00000001:0.0:1787764346.940629:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764346.940631:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876de00 x1874599832715648/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764371 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787764346.940634:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764346.940641:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787764346.940645:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764346.940647:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764346.940649:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764346.940652:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22061
00000400:00000200:0.0:1787764346.940658:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764346.940658:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000400:00000200:0.0:1787764346.940665:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00000040:3.0:1787764346.940665:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00080000:0.0:1787764346.940670:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000400:00000010:3.0:1787764346.940671:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000040:0.0:1787764346.940676:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000200:3.0:1787764346.940677:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832715648, offset 0
00000100:00000001:0.0:1787764346.940680:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764346.940683:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764346.940686:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764346.940690:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:3.0:1787764346.940691:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000020:00000040:0.0:1787764346.940692:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787764346.940695:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764346.940698:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787764346.940699:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:3.0:1787764346.940704:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000001:0.0:1787764346.940705:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787764346.940711:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764346.940713:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000200:3.0:1787764346.940715:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000040:0.0:1787764346.940717:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764346.940721:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:3.0:1787764346.940724:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:0.0:1787764346.940725:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00080000:0.0:1787764346.940730:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764346.940733:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764346.940733:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
02000000:00000001:0.0:1787764346.940736:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764346.940739:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1787764346.940740:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
02000000:00000001:0.0:1787764346.940743:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764346.940747:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:0.0:1787764346.940748:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764346.940752:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:3.0:1787764346.940756:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58826:58826:256:4294967295] 192.168.203.110@tcp LPNI seq info [58826:58826:8:4294967295]
00000008:00000001:0.0:1787764346.940757:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764346.940761:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764346.940766:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cb40.
00000400:00000200:3.0:1787764346.940767:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:0.0:1787764346.940771:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787764346.940774:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764346.940776:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764346.940780:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764346.940782:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764346.940785:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764346.940785:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
02000000:00000010:0.0:1787764346.940791:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000001:0.0:1787764346.940795:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787764346.940796:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00100000:0.0:1787764346.940802:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cb40 x1874599832715904/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000010:3.0:1787764346.940804:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000100:00000001:0.0:1787764346.940812:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764346.940816:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787764346.940816:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764346.940824:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000100:00000040:0.0:1787764346.940825:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008ea72f5a] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cb40 x1874599832715904/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764346.940845:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764346.940848:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787764346.940852:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787764346.940854:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764346.940858:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787764346.940860:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764346.940862:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764346.940864:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764346.940868:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22061
00000800:00000200:3.1:1787764346.940873:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000400:00000200:0.0:1787764346.940875:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:3.1:1787764346.940880:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000400:00000200:0.0:1787764346.940882:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764346.940887:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000100:3.0:1787764346.940892:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000100:00000040:0.0:1787764346.940893:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000010:3.0:1787764346.940897:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000100:00000001:0.0:1787764346.940898:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764346.940901:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764346.940905:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:3.0:1787764346.940906:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:0.0:1787764346.940908:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764346.940910:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000020:00000001:0.0:1787764346.940913:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000400:00000200:3.0:1787764346.940915:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000020:00000040:0.0:1787764346.940917:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787764346.940921:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000020:00000001:0.0:1787764346.940923:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000400:00000200:3.0:1787764346.940925:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000100:00000040:0.0:1787764346.940928:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000400:00000010:3.0:1787764346.940929:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:0.0:1787764346.940931:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764346.940935:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764346.940936:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1787764346.940938:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764346.940941:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00000200:3.0:1787764346.940943:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876de00 x1874599832715648/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764371 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787764346.940945:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764346.940951:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764346.940955:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764346.940959:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764346.940962:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764346.940963:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1787764346.940964:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764346.940969:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000040:3.0:1787764346.940971:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876de00 x1874599832715648/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764371 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000008:00000001:0.0:1787764346.940973:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764346.940977:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764346.940984:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000100:00000001:3.0:1787764346.940985:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787764346.940987:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000100:00000001:3.0:1787764346.940989:0:257575:0:(events.c:68:request_out_callback()) Process leaving
02000000:00000001:0.0:1787764346.940992:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764346.940994:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764346.940996:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764346.940998:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764346.941000:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764346.941000:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764346.941007:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000400:00000200:3.0:1787764346.941008:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764346.941011:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764346.941013:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316735219
00000100:00100000:0.0:1787764346.941019:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832716032/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000010:3.0:1787764346.941022:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40000.
00000100:00000001:0.0:1787764346.941030:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764346.941033:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764346.941033:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000040:0.0:1787764346.941040:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832716032/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941042:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316735219
00000100:00000010:3.0:1787764346.941049:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40600.
00000100:00000001:0.0:1787764346.941054:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764346.941055:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000040:0.0:1787764346.941061:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (22071)
00000100:00000001:3.0:1787764346.941062:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764346.941067:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764346.941070:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764346.941077:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764346/real 1787764346]  req@ffff9751f876de00 x1874599832715648/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764346.941087:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764346.941089:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764346.941091:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764346.941093:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764346.941095:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764346.941097:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764346.941101:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876de00 x1874599832715648/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764346.941111:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876de00 x1874599832715648/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941118:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764346.941120:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.941122:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764346.941125:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764346.941129:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876de00 x1874599832715648/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941155:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764346.941161:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876de00 x1874599832715648/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941169:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764346.941172:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.941173:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764346.941175:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.941178:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764346.941180:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764346.941183:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764346.941188:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764346.941190:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764346.941195:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876de00 x1874599832715648/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764346.941203:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832715648:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764346.941208:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.941210:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764346.941213:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876de00 x1874599832715648/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941226:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764346.941229:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764346.941231:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764346.941234:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764346.941236:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764346.941238:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764346.941240:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764346.941243:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014800.
02000000:00000001:3.0:1787764346.941247:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764346.941249:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764346.941251:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876de00.
00000100:00000001:3.0:1787764346.941256:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764346.941257:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.941260:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.941267:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764346.941269:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764346.941271:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764346.941273:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764346.941274:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764346.941278:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876da40 x1874599832715776/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941285:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764346.941287:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764346.941289:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764346.941291:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764346.941293:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764346.941296:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832715776:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764346.941299:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764346.941301:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764346.941302:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764346.941304:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764346.941307:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764346.941309:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764346.941313:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764346.941317:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764346.941321:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764346.941324:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832715776, portal 10
00000100:00000001:3.0:1787764346.941326:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764346.941327:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049524800 : -115096660026816 : ffff9751f876da40)
00000100:00000040:3.0:1787764346.941334:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876da40 x1874599832715776/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764371 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941341:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764346.941343:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764346.941346:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764346.941348:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832715776, offset 0
00000400:00000200:3.0:1787764346.941352:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764346.941356:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764346.941360:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764346.941372:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764346.941374:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764346.941377:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764346.941379:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764346.941382:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58827:58827:256:4294967295] 192.168.203.110@tcp LPNI seq info [58827:58827:8:4294967295]
00000400:00000200:3.0:1787764346.941386:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764346.941392:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764346.941397:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764346.941401:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764346.941405:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764346.941407:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787764346.941411:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764346.941414:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1787764346.941418:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764346.941420:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764346.941423:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764346.941426:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764346.941429:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764346.941431:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764346.941434:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764346.941437:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876da40 x1874599832715776/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764371 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941446:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764346.941449:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876da40 x1874599832715776/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764371 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941456:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764346.941458:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764346.941461:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764346.941465:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764346.941469:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764346.941471:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316733984
00000100:00000001:3.0:1787764346.941474:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316735219
00000100:00000001:3.0:1787764346.941476:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316733984
00000100:00000001:3.0:1787764346.941478:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316735219
00000100:00000001:3.0:1787764346.941480:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764346.941482:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764346.941484:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764346.941488:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764346/real 1787764346]  req@ffff9751f876da40 x1874599832715776/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764346.941496:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764346.941498:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764346.941499:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764346.941502:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764346.941504:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764346.941506:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764346.941510:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876da40 x1874599832715776/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764346.941518:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876da40 x1874599832715776/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941530:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764346.941532:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.941534:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764346.941535:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764346.941539:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876da40 x1874599832715776/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941545:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764346.941550:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876da40 x1874599832715776/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941557:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764346.941559:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.941561:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764346.941562:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.941564:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764346.941576:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764346.941580:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764346.941583:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764346.941586:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764346.941590:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876da40 x1874599832715776/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764346.941598:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832715776:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764346.941602:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764346.941605:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cb40 x1874599832715904/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941611:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764346.941613:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764346.941616:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764346.941618:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764346.941620:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764346.941622:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832715904:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764346.941626:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764346.941628:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764346.941629:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764346.941631:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764346.941634:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764346.941636:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764346.941640:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764346.941643:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764346.941647:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764346.941649:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832715904, portal 10
00000100:00000001:3.0:1787764346.941652:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764346.941653:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049520960 : -115096660030656 : ffff9751f876cb40)
00000100:00000040:3.0:1787764346.941659:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cb40 x1874599832715904/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764371 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941666:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764346.941668:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764346.941671:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764346.941673:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832715904, offset 0
00000400:00000200:3.0:1787764346.941677:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764346.941681:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764346.941685:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764346.941688:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764346.941690:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764346.941692:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764346.941695:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764346.941697:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58828:58828:256:4294967295] 192.168.203.110@tcp LPNI seq info [58828:58828:8:4294967295]
00000400:00000200:3.0:1787764346.941701:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764346.941708:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764346.941712:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764346.941716:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764346.941719:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764346.941722:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787764346.941726:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764346.941729:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1787764346.941732:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764346.941734:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764346.941737:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764346.941740:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764346.941744:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764346.941745:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764346.941748:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764346.941751:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876cb40 x1874599832715904/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764371 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941759:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764346.941763:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cb40 x1874599832715904/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764371 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941769:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764346.941770:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764346.941774:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764346.941777:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764346.941780:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764346.941782:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316733972
00000100:00000001:3.0:1787764346.941784:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316735219
00000100:00000001:3.0:1787764346.941786:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316733972
00000100:00000001:3.0:1787764346.941789:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316735219
00000100:00000001:3.0:1787764346.941791:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764346.941793:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764346.941795:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764346.941799:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764346/real 1787764346]  req@ffff9751f876cb40 x1874599832715904/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764346.941807:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764346.941808:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764346.941810:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764346.941813:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764346.941814:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764346.941816:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764346.941820:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cb40 x1874599832715904/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764346.941828:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cb40 x1874599832715904/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941849:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764346.941850:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.941852:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764346.941853:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764346.941857:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cb40 x1874599832715904/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941864:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764346.941868:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cb40 x1874599832715904/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941875:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764346.941877:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.941878:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764346.941879:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.941881:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764346.941882:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764346.941885:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764346.941888:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764346.941891:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764346.941895:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cb40 x1874599832715904/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764346.941903:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832715904:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764346.941907:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764346.941909:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832716032/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941916:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764346.941917:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764346.941919:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764346.941921:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764346.941923:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764346.941925:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832716032:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764346.941929:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764346.941930:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764346.941932:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764346.941934:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764346.941937:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764346.941939:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764346.941942:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764346.941944:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764346.941948:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764346.941950:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832716032, portal 4
00000100:00000001:3.0:1787764346.941952:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764346.941954:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00000040:3.0:1787764346.941959:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832716032/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764371 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.941965:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764346.941968:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764346.941970:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764346.941972:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832716032, offset 0
00000400:00000200:3.0:1787764346.941977:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764346.941980:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764346.941984:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764346.941987:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764346.941989:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764346.941992:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764346.941994:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764346.941997:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58829:58829:256:4294967295] 192.168.203.110@tcp LPNI seq info [58829:58829:8:4294967295]
00000400:00000200:3.0:1787764346.942001:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764346.942007:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764346.942011:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764346.942014:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764346.942018:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764346.942020:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000200:3.1:1787764346.942024:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764346.942026:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 7 secs later)
00000800:00000100:3.0:1787764346.942029:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764346.942031:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764346.942033:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764346.942036:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764346.942039:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764346.942041:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764346.942043:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764346.942046:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e580 x1874599832716032/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764371 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.942053:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764346.942056:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832716032/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764371 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.942063:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764346.942064:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764346.942067:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764346.942070:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764346.942072:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764346.942075:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316733989
00000100:00000001:3.0:1787764346.942077:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316735219
00000100:00000001:3.0:1787764346.942079:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316733989
00000100:00000001:3.0:1787764346.942081:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316735219
00000100:00000001:3.0:1787764346.942083:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764346.942084:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764346.942086:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764346.942089:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764346/real 1787764346]  req@ffff9751f876e580 x1874599832716032/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764346.942095:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764346.942096:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764346.942098:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764346.942100:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764346.942101:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764346.942103:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764346.942106:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832716032/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764346.942113:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832716032/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.942118:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764346.942119:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.942120:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764346.942122:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764346.942125:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832716032/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.942131:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764346.942148:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832716032/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.942155:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764346.942158:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.942159:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764346.942161:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.942162:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764346.942164:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764346.942166:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764346.942170:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764346.942172:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764346.942176:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832716032/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764346.942184:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832716032:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764346.942187:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.942189:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764346.942192:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876da40 x1874599832715776/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.942199:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764346.942201:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764346.942203:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764346.942205:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764346.942206:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764346.942208:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764346.942210:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764346.942212:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016800.
02000000:00000001:3.0:1787764346.942215:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764346.942216:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764346.942217:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876da40.
00000100:00000001:3.0:1787764346.942220:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764346.942221:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.942222:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764346.942226:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cb40 x1874599832715904/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.942238:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764346.942239:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764346.942240:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764346.942243:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764346.942244:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764346.942245:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764346.942247:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764346.942249:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:3.0:1787764346.942251:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764346.942252:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764346.942254:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cb40.
00000100:00000001:3.0:1787764346.942256:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764346.942258:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.942259:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764346.942262:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832716032/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764371 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764346.942269:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764346.942270:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764346.942271:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764346.942274:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764346.942275:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764346.942276:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764346.942278:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764346.942279:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017000.
02000000:00000001:3.0:1787764346.942282:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764346.942284:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764346.942285:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:3.0:1787764346.942287:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764346.942288:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.942290:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764346.942294:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764346.942295:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764346.942297:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764346.942298:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764347.580256:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764347.580263:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764347.580267:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764347.580269:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764347.580271:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764347.580277:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764347.580277:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764347.580282:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764347.580284:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764347.580286:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764347.580288:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764347.580289:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764347.580292:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764347.580293:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764347.580295:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764347.580298:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764347.580302:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764347.580305:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764347.580308:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764347.580313:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764347.580314:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764347.580369:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764347.580373:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764347.580375:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764347.964227:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764347.964238:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764347.964241:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764347.964248:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764347.964252:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764347.964256:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764347.964258:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764347.964262:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764347.964265:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764348.220219:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316735219, at 4316735232
00000100:00000010:3.0:1787764348.220240:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40600.
00000100:00000040:2.0:1787764348.220251:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316735219, at 4316735232
00000100:00000010:2.0:1787764348.220258:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40000.
00010000:00000001:3.0:1787764348.604235:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764348.604239:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764348.604245:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764348.604245:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764348.604247:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764348.604253:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764348.604256:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764348.604256:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764348.604259:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764348.604261:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764348.604264:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764348.604265:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764348.604267:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764348.604268:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764348.604271:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764348.604305:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764348.604310:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764348.604313:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764348.988282:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764348.988291:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764348.988294:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764348.988301:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764348.988306:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764348.988310:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764348.988313:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764348.988315:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764348.988319:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764349.628237:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764349.628243:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764349.628246:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764349.628248:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764349.628255:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764349.628256:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764349.628260:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764349.628260:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764349.628266:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764349.628270:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764349.628275:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764349.628278:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764349.628282:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764349.628284:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764349.628288:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764349.628304:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764349.628309:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764349.628312:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764350.012255:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764350.012263:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764350.012265:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764350.012269:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764350.012273:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764350.012276:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764350.012278:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764350.012280:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764350.012282:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764350.652245:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764350.652252:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764350.652262:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764350.652263:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764350.652266:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764350.652273:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764350.652275:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764350.652278:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764350.652280:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764350.652290:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764350.652298:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764350.652303:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764350.652307:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764350.652311:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764350.652315:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764350.652341:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764350.652345:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764350.652348:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764351.036262:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764351.036270:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764351.036272:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764351.036277:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764351.036280:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764351.036284:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764351.036286:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764351.036288:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764351.036291:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764351.676278:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764351.676279:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764351.676288:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764351.676289:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764351.676294:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764351.676298:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764351.676303:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764351.676304:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764351.676310:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764351.676313:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764351.676314:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764351.676321:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764351.676326:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764351.676330:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764351.676334:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764351.676374:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764351.676379:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764351.676383:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764352.060272:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764352.060280:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764352.060285:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764352.060292:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:0.0:1787764352.060299:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764352.060300:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764352.060305:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787764352.060307:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:3.0:1787764352.060310:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764352.060311:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00000001:3.0:1787764352.060314:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00080000:0.0:1787764352.060317:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764352.060318:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764352.060324:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764352.060333:0:304484:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 22066
00000400:00000200:0.0:1787764352.060343:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764352.060350:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764352.060353:0:304484:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764352.060358:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764352.060362:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764352.060365:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764352.060367:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764352.060372:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764352.060374:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764352.060377:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764352.060380:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764352.060383:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764352.060386:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787764352.060389:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764352.060391:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764352.060394:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764352.060396:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764352.060399:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787764352.060403:0:304484:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764352.060406:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764352.060409:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764352.060413:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764352.060416:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764352.060428:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764352.060430:0:304484:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764352.060440:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000020:00000040:0.0:1787764352.060443:0:304484:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764352.060447:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764352.060449:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764352.060453:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764352.060455:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764352.060459:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764352.060468:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787764352.060474:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764352.060485:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832716160/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764352.060498:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764352.060502:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764352.060507:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832716160/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764352.060533:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764352.060536:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764352.060540:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764352.060541:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764352.060543:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764352.060546:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764352.060549:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22066
00000400:00000200:0.0:1787764352.060554:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764352.060559:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764352.060562:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764352.060566:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764352.060567:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787764352.060569:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764352.060572:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:0.0:1787764352.060574:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764352.060575:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1787764352.060580:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000040:3.0:1787764352.060584:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832716160/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787764352.060585:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764352.060588:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787764352.060593:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764352.060597:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000001:3.0:1787764352.060598:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764352.060600:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764352.060605:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:3.0:1787764352.060605:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764352.060608:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000020:00000001:0.0:1787764352.060610:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
02000000:00000001:3.0:1787764352.060612:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764352.060619:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832716160:192.168.203.110@tcp:250:kworker.0
00000100:00000040:0.0:1787764352.060622:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764352.060624:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1787764352.060627:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:3.0:1787764352.060630:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:0.0:1787764352.060632:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:3.0:1787764352.060635:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764352.060638:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000001:3.0:1787764352.060640:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:0.0:1787764352.060644:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787764352.060648:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000010:3.0:1787764352.060650:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
00000100:00000001:0.0:1787764352.060652:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764352.060654:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764352.060656:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1787764352.060663:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787764352.060664:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764352.060669:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764352.060669:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000100:00000001:0.0:1787764352.060675:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000010:3.0:1787764352.060679:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000008:00100000:0.0:1787764352.060680:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787764352.060686:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764352.060689:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764352.060689:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832716160, portal 25
00000100:00000001:3.0:1787764352.060695:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764352.060698:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000100:00000010:0.0:1787764352.060699:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787764352.060705:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000100:00000040:3.0:1787764352.060708:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832716160/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764377 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787764352.060710:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764352.060713:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764352.060719:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764352.060722:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000001:3.0:1787764352.060724:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
02000000:00000001:0.0:1787764352.060726:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764352.060730:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
02000000:00000010:0.0:1787764352.060734:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000400:00000010:3.0:1787764352.060736:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000001:0.0:1787764352.060742:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764352.060742:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832716160, offset 0
00000100:00100000:0.0:1787764352.060754:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832716288/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000400:00000200:3.0:1787764352.060755:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764352.060766:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:0.0:1787764352.060768:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764352.060772:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764352.060775:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000040:0.0:1787764352.060781:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832716288/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1787764352.060785:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764352.060805:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764352.060810:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000100:00000001:0.0:1787764352.060812:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764352.060816:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00080000:0.0:1787764352.060818:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:3.0:1787764352.060824:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58830:58830:256:4294967295] 192.168.203.110@tcp LPNI seq info [58830:58830:8:4294967295]
00000100:00080000:0.0:1787764352.060826:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787764352.060830:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000400:00000200:3.0:1787764352.060834:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:0.0:1787764352.060835:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764352.060840:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764352.060848:0:304484:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22066
00000400:00000200:3.0:1787764352.060853:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000400:00000200:0.0:1787764352.060858:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000800:00000200:3.0:1787764352.060865:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000400:00000200:0.0:1787764352.060866:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764352.060872:0:304484:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000800:00000010:3.0:1787764352.060875:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000100:00000040:0.0:1787764352.060876:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764352.060881:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:3.0:1787764352.060885:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000040:0.0:1787764352.060888:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000800:00000200:3.0:1787764352.060890:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000100:00000001:0.0:1787764352.060893:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000200:3.1:1787764352.060898:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787764352.060900:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000800:00000200:3.1:1787764352.060902:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000020:00000040:0.0:1787764352.060903:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000800:00000100:3.0:1787764352.060907:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000020:00000001:0.0:1787764352.060909:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000800:00000010:3.0:1787764352.060910:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000020:00000040:0.0:1787764352.060915:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000400:00000200:3.0:1787764352.060917:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000020:00000001:0.0:1787764352.060921:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:3.0:1787764352.060923:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000020:00000001:0.0:1787764352.060928:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:3.0:1787764352.060930:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764352.060934:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000040:0.0:1787764352.060937:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764352.060938:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000001:0.0:1787764352.060943:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:3.0:1787764352.060944:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e1c0 x1874599832716160/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764377 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764352.060949:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764352.060956:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764352.060959:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000020:00000040:0.0:1787764352.060963:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000100:00000040:3.0:1787764352.060964:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832716160/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764377 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787764352.060972:0:304484:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764352.060974:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764352.060977:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787764352.060980:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764352.060983:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764352.060985:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1787764352.060989:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787764352.060990:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764352.060992:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316740339
02000000:00000001:0.0:1787764352.060996:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:3.0:1787764352.060998:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40600.
00000100:00000001:0.0:1787764352.061001:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764352.061005:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000100:00000001:3.0:1787764352.061006:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000008:00000001:0.0:1787764352.061010:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764352.061014:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764352.061015:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316740339
00000100:00000010:0.0:1787764352.061020:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000100:00000010:3.0:1787764352.061020:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40de0.
00000100:00000001:3.0:1787764352.061025:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000020:00000040:0.0:1787764352.061026:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000100:00000001:3.0:1787764352.061029:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764352.061033:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764352.061034:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000100:00000001:3.0:1787764352.061035:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
02000000:00000001:0.0:1787764352.061039:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764352.061040:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764352/real 1787764352]  req@ffff9751f876e1c0 x1874599832716160/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764352.061044:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764352.061048:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000001:3.0:1787764352.061051:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
02000000:00000001:0.0:1787764352.061052:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000001:3.0:1787764352.061053:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764352.061055:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764352.061058:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764352.061059:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
02000000:00000010:0.0:1787764352.061061:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000100:00000001:3.0:1787764352.061061:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764352.061066:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832716160/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764352.061069:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764352.061080:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832716160/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787764352.061082:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832716416/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764352.061092:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764352.061095:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764352.061099:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764352.061101:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764352.061103:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00100000:3.0:1787764352.061106:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832716160/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764352.061108:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764352.061116:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787764352.061120:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832716416/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:3.0:1787764352.061123:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832716160/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.061158:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764352.061160:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764352.061161:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764352.061162:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764352.061164:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764352.061167:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764352.061172:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:0.0:1787764352.061175:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764352.061176:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000040:0.0:1787764352.061181:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:0.0:1787764352.061188:0:304484:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00000001:3.0:1787764352.061189:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00080000:0.0:1787764352.061193:0:304484:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000040:3.0:1787764352.061195:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832716160/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787764352.061202:0:304484:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787764352.061207:0:304484:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00100000:3.0:1787764352.061207:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832716160:192.168.203.110@tcp:250:kworker.0
00000100:00080000:0.0:1787764352.061211:0:304484:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764352.061213:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764352.061216:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000001:0.0:1787764352.061218:0:304484:0:(import.c:485:import_select_connection()) Process entered
00000100:00000040:3.0:1787764352.061220:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832716160/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787764352.061226:0:304484:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22066
00000100:00000001:3.0:1787764352.061229:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764352.061233:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
00000400:00000200:0.0:1787764352.061236:0:304484:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000010:3.0:1787764352.061236:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764352.061242:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764352.061244:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000400:00000200:0.0:1787764352.061245:0:304484:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000020:00000040:3.0:1787764352.061246:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764352.061251:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
00000100:00080000:0.0:1787764352.061252:0:304484:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
02000000:00000010:3.0:1787764352.061253:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014400.
02000000:00000001:3.0:1787764352.061259:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
00000100:00000040:0.0:1787764352.061261:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:3.0:1787764352.061261:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764352.061264:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:0.0:1787764352.061267:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764352.061269:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764352.061272:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:0.0:1787764352.061273:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764352.061274:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764352.061280:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764352.061283:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000020:00000001:0.0:1787764352.061284:0:304484:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1787764352.061286:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787764352.061289:0:304484:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000100:00000001:3.0:1787764352.061289:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764352.061292:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1787764352.061294:0:304484:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:3.0:1787764352.061294:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000040:0.0:1787764352.061300:0:304484:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000100:00000040:3.0:1787764352.061300:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832716288/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787764352.061307:0:304484:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:3.0:1787764352.061311:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000020:00000001:0.0:1787764352.061314:0:304484:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:3.0:1787764352.061314:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764352.061319:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764352.061321:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1787764352.061324:0:304484:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
02000000:00000001:3.0:1787764352.061324:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764352.061330:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832716288:192.168.203.110@tcp:38:kworker.0
00000100:00000001:0.0:1787764352.061331:0:304484:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764352.061335:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764352.061338:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
00000100:00000040:0.0:1787764352.061339:0:304484:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:3.0:1787764352.061340:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764352.061344:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000100:00000001:0.0:1787764352.061347:0:304484:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
02000000:00000010:3.0:1787764352.061348:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764352.061353:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787764352.061354:0:304484:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000400:00000200:3.0:1787764352.061358:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764352.061362:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000100:00080000:0.0:1787764352.061363:0:304484:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000400:00000010:3.0:1787764352.061365:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000001:0.0:1787764352.061369:0:304484:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764352.061370:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832716288, portal 10
02000000:00000001:0.0:1787764352.061375:0:304484:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764352.061375:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764352.061378:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
02000000:00000001:0.0:1787764352.061382:0:304484:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:3.0:1787764352.061386:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832716288/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764377 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000001:0.0:1787764352.061388:0:304484:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764352.061395:0:304484:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764352.061398:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000008:00100000:0.0:1787764352.061400:0:304484:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00000040:3.0:1787764352.061401:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764352.061403:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000008:00000001:0.0:1787764352.061407:0:304484:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764352.061407:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832716288, offset 0
00000100:00000001:0.0:1787764352.061413:0:304484:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764352.061416:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764352.061421:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000010:0.0:1787764352.061422:0:304484:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
00000400:00000200:3.0:1787764352.061425:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000020:00000040:0.0:1787764352.061428:0:304484:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
00000400:00000200:3.0:1787764352.061428:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764352.061433:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
02000000:00000001:0.0:1787764352.061435:0:304484:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
00000400:00000200:3.0:1787764352.061436:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
02000000:00000001:0.0:1787764352.061439:0:304484:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764352.061440:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000100:00000001:0.0:1787764352.061445:0:304484:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000400:00000200:3.0:1787764352.061445:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58831:58831:256:4294967295] 192.168.203.110@tcp LPNI seq info [58831:58831:8:4294967295]
02000000:00000001:0.0:1787764352.061450:0:304484:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000400:00000200:3.0:1787764352.061450:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
02000000:00000001:0.0:1787764352.061454:0:304484:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764352.061458:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
02000000:00000010:0.0:1787764352.061462:0:304484:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000800:00000200:3.0:1787764352.061464:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787764352.061469:0:304484:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1787764352.061470:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764352.061476:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764352.061479:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000100:00100000:0.0:1787764352.061482:0:304484:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832716544/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000800:00000200:3.1:1787764352.061484:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764352.061489:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1787764352.061498:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764352.061500:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764352.061504:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000100:00000001:0.0:1787764352.061506:0:304484:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000400:00000200:3.0:1787764352.061510:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000100:00000001:0.0:1787764352.061512:0:304484:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764352.061513:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764352.061516:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764352.061519:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:0.0:1787764352.061523:0:304484:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832716544/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764352.061524:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876c000 x1874599832716288/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764377 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.061545:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764352.061549:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832716288/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764377 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764352.061552:0:304484:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764352.061558:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764352.061560:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000040:0.0:1787764352.061561:0:304484:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (22076)
00000100:00000001:3.0:1787764352.061563:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764352.061567:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764352.061569:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764352.061571:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316739104
00000100:00000001:3.0:1787764352.061573:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316740339
00000100:00000001:3.0:1787764352.061575:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316739104
00000100:00000001:3.0:1787764352.061577:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316740339
00000100:00000001:3.0:1787764352.061579:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764352.061580:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764352.061581:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764352.061584:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764352/real 1787764352]  req@ffff9751f876c000 x1874599832716288/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764352.061589:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764352.061590:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764352.061591:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764352.061592:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764352.061594:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764352.061595:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764352.061596:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832716288/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764352.061602:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832716288/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.061606:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764352.061607:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764352.061608:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764352.061609:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764352.061611:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832716288/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.061615:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764352.061618:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832716288/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.061622:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764352.061623:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764352.061625:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764352.061626:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764352.061627:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764352.061636:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764352.061638:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764352.061640:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764352.061642:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764352.061644:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832716288/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764352.061649:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832716288:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764352.061651:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764352.061653:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f0c0 x1874599832716416/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.061657:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764352.061658:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764352.061659:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764352.061660:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764352.061662:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764352.061664:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832716416:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764352.061666:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764352.061667:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764352.061668:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764352.061669:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764352.061671:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764352.061673:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764352.061674:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764352.061676:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764352.061677:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764352.061679:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832716416, portal 10
00000100:00000001:3.0:1787764352.061680:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764352.061682:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049530560 : -115096660021056 : ffff9751f876f0c0)
00000100:00000040:3.0:1787764352.061685:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f0c0 x1874599832716416/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764377 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.061690:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764352.061691:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764352.061692:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764352.061694:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832716416, offset 0
00000400:00000200:3.0:1787764352.061696:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764352.061698:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764352.061700:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764352.061702:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764352.061704:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764352.061705:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764352.061707:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764352.061709:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58832:58832:256:4294967295] 192.168.203.110@tcp LPNI seq info [58832:58832:8:4294967295]
00000400:00000200:3.0:1787764352.061711:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764352.061715:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764352.061718:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764352.061720:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764352.061722:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764352.061724:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787764352.061727:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764352.061728:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1787764352.061731:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764352.061732:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764352.061734:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764352.061736:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764352.061738:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764352.061739:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764352.061741:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764352.061743:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f0c0 x1874599832716416/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764377 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.061748:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764352.061750:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f0c0 x1874599832716416/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764377 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.061754:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764352.061755:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764352.061757:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764352.061759:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764352.061760:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764352.061762:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316739092
00000100:00000001:3.0:1787764352.061763:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316740339
00000100:00000001:3.0:1787764352.061765:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316739092
00000100:00000001:3.0:1787764352.061766:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316740339
00000100:00000001:3.0:1787764352.061768:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764352.061769:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764352.061770:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764352.061772:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764352/real 1787764352]  req@ffff9751f876f0c0 x1874599832716416/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764352.061776:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764352.061777:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764352.061778:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764352.061779:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764352.061780:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764352.061781:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764352.061783:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f0c0 x1874599832716416/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764352.061788:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f0c0 x1874599832716416/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.061792:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764352.061808:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764352.061809:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764352.061810:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764352.061814:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f0c0 x1874599832716416/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.061819:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764352.061821:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f0c0 x1874599832716416/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.061826:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764352.061827:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764352.061828:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764352.061829:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764352.061830:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764352.061831:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764352.061832:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764352.061834:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764352.061835:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764352.061838:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f0c0 x1874599832716416/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764352.061842:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832716416:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764352.061845:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764352.061846:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764352.061848:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832716288/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.061852:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764352.061853:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764352.061855:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764352.061856:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764352.061857:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764352.061858:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764352.061859:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764352.061861:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017800.
02000000:00000001:3.0:1787764352.061864:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764352.061865:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764352.061866:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:3.0:1787764352.061868:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764352.061869:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764352.061870:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764352.061872:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f0c0 x1874599832716416/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.061876:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764352.061877:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764352.061878:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764352.061879:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764352.061880:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764352.061881:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764352.061882:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764352.061883:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016c00.
02000000:00000001:3.0:1787764352.061885:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764352.061885:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764352.061886:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f0c0.
00000100:00000001:3.0:1787764352.061888:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764352.061888:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764352.061889:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764352.061891:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764352.061892:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764352.061893:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764352.061894:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764352.061895:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764352.061897:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832716544/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.061902:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764352.061902:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764352.061904:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764352.061905:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764352.061907:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764352.061908:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832716544:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764352.061911:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764352.061912:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764352.061914:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764352.061915:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764352.061917:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764352.061918:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764352.061920:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764352.061922:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000400:00000010:3.0:1787764352.061925:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
00000100:00000200:3.0:1787764352.061927:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832716544, portal 4
00000100:00000001:3.0:1787764352.061928:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764352.061929:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
00000100:00000040:3.0:1787764352.061933:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832716544/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764377 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.061937:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764352.061939:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764352.061940:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000200:3.0:1787764352.061942:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832716544, offset 0
00000400:00000200:3.0:1787764352.061944:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764352.061947:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764352.061949:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764352.061950:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764352.061952:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764352.061954:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764352.061955:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764352.061957:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58833:58833:256:4294967295] 192.168.203.110@tcp LPNI seq info [58833:58833:8:4294967295]
00000400:00000200:3.0:1787764352.061960:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764352.061964:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764352.061967:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764352.061969:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000800:00000200:3.0:1787764352.061971:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.0:1787764352.061973:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000200:3.1:1787764352.061976:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764352.061978:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 2 secs later)
00000800:00000100:3.0:1787764352.061980:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764352.061982:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:3.0:1787764352.061984:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:3.0:1787764352.061986:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:3.0:1787764352.061988:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:3.0:1787764352.061989:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:3.0:1787764352.061991:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:3.0:1787764352.061993:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876ed00 x1874599832716544/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764377 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.061998:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764352.061999:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832716544/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764377 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.062004:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764352.062005:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764352.062006:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764352.062008:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764352.062010:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764352.062011:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316739109
00000100:00000001:3.0:1787764352.062013:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316740339
00000100:00000001:3.0:1787764352.062015:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316739109
00000100:00000001:3.0:1787764352.062016:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316740339
00000100:00000001:3.0:1787764352.062017:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764352.062019:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764352.062020:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764352.062022:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764352/real 1787764352]  req@ffff9751f876ed00 x1874599832716544/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764352.062026:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764352.062027:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764352.062029:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764352.062030:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764352.062031:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764352.062032:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764352.062034:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832716544/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764352.062040:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832716544/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.062043:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764352.062044:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764352.062045:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764352.062046:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764352.062048:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832716544/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.062052:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764352.062054:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832716544/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.062058:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764352.062060:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764352.062061:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764352.062062:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764352.062063:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764352.062064:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764352.062066:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764352.062068:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764352.062069:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764352.062071:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832716544/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764352.062076:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832716544:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764352.062078:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764352.062080:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764352.062081:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832716544/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764377 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764352.062086:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764352.062086:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764352.062088:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764352.062089:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764352.062090:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764352.062090:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:3.0:1787764352.062091:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764352.062092:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
02000000:00000001:3.0:1787764352.062093:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764352.062094:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764352.062095:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:3.0:1787764352.062097:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764352.062098:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764352.062099:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764352.062101:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764352.062102:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764352.062103:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764352.062104:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764352.700264:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764352.700270:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764352.700270:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764352.700282:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764352.700283:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764352.700288:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764352.700289:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764352.700302:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764352.700305:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764352.700305:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764352.700315:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764352.700321:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764352.700325:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764352.700329:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764352.700335:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764352.700349:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764352.700353:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764352.700357:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764353.084238:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764353.084245:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764353.084246:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764353.084250:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764353.084254:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764353.084256:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764353.084259:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764353.084261:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764353.084264:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764353.340260:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316740339, at 4316740352
00000100:00000010:3.0:1787764353.340275:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40de0.
00000100:00000040:2.0:1787764353.340300:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316740339, at 4316740352
00000100:00000010:2.0:1787764353.340308:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40600.
00000100:00000001:2.0:1787764353.724232:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764353.724234:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764353.724241:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764353.724243:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764353.724248:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764353.724250:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764353.724254:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764353.724255:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764353.724262:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764353.724262:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764353.724267:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764353.724268:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764353.724273:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764353.724274:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764353.724277:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764353.724279:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764353.724281:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764353.724284:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764353.724286:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764353.724287:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764353.724292:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764353.724349:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764353.724354:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764353.724358:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764354.108234:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764354.108242:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764354.108244:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764354.108249:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764354.108252:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764354.108255:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764354.108257:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764354.108259:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764354.108262:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000008:00000020:0.0:1787764354.620204:0:304600:0:(osc_request.c:747:osc_update_next_shrink()) next time 23274 to shrink grant
00000100:00000001:2.0:1787764354.748251:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764354.748252:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764354.748257:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764354.748259:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764354.748263:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764354.748264:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764354.748266:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764354.748272:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764354.748273:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764354.748276:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764354.748279:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764354.748293:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764354.748298:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764354.748301:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764354.748304:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764354.748306:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764354.748310:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764354.748314:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764355.132220:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764355.132227:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764355.132229:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764355.132234:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764355.132238:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764355.132241:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764355.132242:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764355.132245:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764355.132247:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764355.772203:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764355.772210:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764355.772213:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764355.772219:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764355.772220:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764355.772223:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764355.772224:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764355.772226:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764355.772227:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764355.772229:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764355.772231:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764355.772232:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764355.772233:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764355.772235:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764355.772236:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764355.772238:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764355.772241:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764355.772246:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764355.772248:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764355.772253:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764355.772254:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764355.772309:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764355.772312:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764355.772316:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:1.0:1787764356.156246:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:1.0:1787764356.156252:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764356.156254:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764356.156258:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:1.0:1787764356.156262:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764356.156273:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764356.156277:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764356.156281:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764356.156288:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764356.796247:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764356.796255:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764356.796256:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764356.796258:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764356.796261:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764356.796265:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764356.796269:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764356.796270:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764356.796273:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764356.796275:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764356.796280:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764356.796280:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764356.796282:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764356.796285:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764356.796288:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764356.796330:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764356.796335:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764356.796339:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00080000:0.0:1787764357.180214:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764357.180222:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787764357.180225:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764357.180229:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764357.180233:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764357.180239:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 22071
00000400:00000200:0.0:1787764357.180246:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764357.180252:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764357.180255:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764357.180258:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764357.180260:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764357.180263:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764357.180265:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764357.180269:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000100:00000001:3.0:1787764357.180269:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000020:00000040:0.0:1787764357.180271:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764357.180275:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764357.180278:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764357.180280:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764357.180282:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000001:3.0:1787764357.180285:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000040:0.0:1787764357.180286:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764357.180290:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764357.180291:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764357.180295:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764357.180295:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787764357.180302:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00100000:3.0:1787764357.180303:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00080000:0.0:1787764357.180306:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764357.180310:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764357.180310:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
02000000:00000001:0.0:1787764357.180312:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:3.0:1787764357.180315:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787764357.180316:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764357.180322:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764357.180322:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764357.180323:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:3.0:1787764357.180326:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764357.180327:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764357.180334:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764357.180339:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f840.
00000020:00000040:0.0:1787764357.180342:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764357.180345:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764357.180346:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764357.180349:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764357.180351:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764357.180354:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764357.180361:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015400.
00000100:00000001:0.0:1787764357.180366:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764357.180373:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f840 x1874599832716672/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764357.180390:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764357.180393:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764357.180397:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a9b5eb67] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f840 x1874599832716672/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764357.180411:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764357.180413:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764357.180416:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764357.180417:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764357.180418:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764357.180421:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764357.180424:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22071
00000400:00000200:0.0:1787764357.180428:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764357.180432:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764357.180435:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764357.180439:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764357.180442:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764357.180445:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764357.180446:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:0.0:1787764357.180448:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764357.180452:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000020:00000001:0.0:1787764357.180453:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764357.180458:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000100:00000001:3.0:1787764357.180458:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1787764357.180462:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764357.180466:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000020:00000001:0.0:1787764357.180470:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:3.0:1787764357.180471:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f840 x1874599832716672/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787764357.180474:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787764357.180483:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764357.180487:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764357.180490:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764357.180494:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764357.180494:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764357.180498:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
00000020:00000040:0.0:1787764357.180499:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00080000:0.0:1787764357.180505:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
02000000:00000001:3.0:1787764357.180507:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1787764357.180509:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764357.180513:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764357.180517:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764357.180522:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00100000:3.0:1787764357.180523:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832716672:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764357.180530:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:0.0:1787764357.180535:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764357.180535:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:0.0:1787764357.180541:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764357.180541:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764357.180547:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:3.0:1787764357.180548:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000008:00100000:0.0:1787764357.180552:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787764357.180556:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:3.0:1787764357.180556:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8611c00.
00000100:00000001:0.0:1787764357.180563:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764357.180563:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764357.180571:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876cf00.
00000400:00000200:3.0:1787764357.180576:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:0.0:1787764357.180577:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787764357.180582:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764357.180584:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764357.180585:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc2342c0.
00000100:00000001:0.0:1787764357.180586:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764357.180589:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764357.180591:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:3.0:1787764357.180596:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d326e8.
02000000:00000010:0.0:1787764357.180603:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016000.
00000100:00000200:3.0:1787764357.180605:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832716672, portal 25
00000100:00000001:0.0:1787764357.180608:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764357.180610:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764357.180613:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049532480 : -115096660019136 : ffff9751f876f840)
00000100:00100000:0.0:1787764357.180615:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876cf00 x1874599832716800/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764357.180625:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:3.0:1787764357.180625:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f840 x1874599832716672/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764382 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764357.180628:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764357.180635:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bc7a06b] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876cf00 x1874599832716800/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764357.180644:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1787764357.180649:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764357.180650:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00080000:0.0:1787764357.180652:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764357.180655:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000400:00000010:3.0:1787764357.180656:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32660.
00000100:00000001:0.0:1787764357.180657:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764357.180660:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764357.180663:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000200:3.0:1787764357.180664:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832716672, offset 0
00000100:00080000:0.0:1787764357.180667:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22071
00000400:00000200:0.0:1787764357.180672:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764357.180676:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:3.0:1787764357.180677:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00080000:0.0:1787764357.180678:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764357.180684:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764357.180685:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764357.180687:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764357.180689:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764357.180693:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:3.0:1787764357.180693:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000020:00000040:0.0:1787764357.180696:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787764357.180698:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764357.180700:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000020:00000001:0.0:1787764357.180701:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000020:00000001:0.0:1787764357.180704:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000400:00000200:3.0:1787764357.180705:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00000040:0.0:1787764357.180709:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764357.180713:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764357.180714:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764357.180717:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000400:00000200:3.0:1787764357.180717:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000020:00000040:0.0:1787764357.180722:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000400:00000200:3.0:1787764357.180727:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000100:00080000:0.0:1787764357.180728:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764357.180732:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764357.180735:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
02000000:00000001:0.0:1787764357.180736:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:3.0:1787764357.180740:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
02000000:00000001:0.0:1787764357.180741:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:3.0:1787764357.180746:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58834:58834:256:4294967295] 192.168.203.110@tcp LPNI seq info [58834:58834:8:4294967295]
02000000:00000001:0.0:1787764357.180748:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764357.180754:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:3.0:1787764357.180755:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000008:00100000:0.0:1787764357.180769:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000008:00000001:0.0:1787764357.180774:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764357.180780:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764357.180780:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00000010:0.0:1787764357.180786:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e940.
00000020:00000040:0.0:1787764357.180791:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000800:00000200:3.0:1787764357.180791:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
02000000:00000001:0.0:1787764357.180797:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764357.180799:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:3.0:1787764357.180800:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186a00.
00000100:00000001:0.0:1787764357.180801:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764357.180804:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764357.180807:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000800:00000200:3.0:1787764357.180811:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
02000000:00000010:0.0:1787764357.180813:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016c00.
00000800:00000200:3.1:1787764357.180818:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787764357.180819:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764357.180825:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e940 x1874599832716928/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764357.180831:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:3.0:1787764357.180832:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764357.180833:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764357.180841:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764357.180844:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316745459
00000100:00000040:0.0:1787764357.180847:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b28bee0e] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e940 x1874599832716928/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1787764357.180852:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40de0.
00000100:00000001:0.0:1787764357.180859:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764357.180862:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:3.0:1787764357.180864:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000040:0.0:1787764357.180865:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787764357.180870:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764357.180872:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000001:0.0:1787764357.180873:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764357.180875:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:3.0:1787764357.180875:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316745459
00000100:00000001:0.0:1787764357.180879:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764357.180882:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22071
00000100:00000010:3.0:1787764357.180883:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e6d40120.
00000400:00000200:0.0:1787764357.180888:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764357.180891:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000400:00000200:0.0:1787764357.180892:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764357.180897:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000001:3.0:1787764357.180899:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1787764357.180900:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764357.180904:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:3.0:1787764357.180907:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764357.180908:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764357.180912:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:3.0:1787764357.180913:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764357.180915:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764357.180918:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000100:00000001:3.0:1787764357.180918:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000020:00000001:0.0:1787764357.180920:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764357.180923:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787764357.180924:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:3.0:1787764357.180926:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764357.180929:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000020:00000001:0.0:1787764357.180930:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000001:3.0:1787764357.180931:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764357.180935:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000040:0.0:1787764357.180936:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:3.0:1787764357.180938:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000001:0.0:1787764357.180939:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764357.180945:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000040:3.0:1787764357.180946:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876cf00 x1874599832716800/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764357.180947:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764357.180953:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787764357.180959:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:3.0:1787764357.180962:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:0.0:1787764357.180963:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764357.180965:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764357.180966:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:3.0:1787764357.180968:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:0.0:1787764357.180970:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764357.180971:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764357.180975:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764357.180976:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764357.180979:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764357.180983:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000100:00100000:3.0:1787764357.180983:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832716800:192.168.203.110@tcp:38:kworker.0
00000008:00000001:0.0:1787764357.180990:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764357.180990:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1787764357.180995:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764357.180995:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764357.180998:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764357.181001:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876ed00.
02000000:00000001:3.0:1787764357.181002:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
00000020:00000040:0.0:1787764357.181006:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000010:3.0:1787764357.181007:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8612000.
02000000:00000001:0.0:1787764357.181008:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:3.0:1787764357.181011:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764357.181012:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764357.181017:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764357.181018:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764357.181021:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764357.181021:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764357.181025:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc234f20.
02000000:00000010:0.0:1787764357.181028:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017800.
00000400:00000010:3.0:1787764357.181031:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32e58.
00000100:00000001:0.0:1787764357.181032:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764357.181036:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832716800, portal 10
00000100:00100000:0.0:1787764357.181040:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876ed00 x1874599832717056/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:3.0:1787764357.181041:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764357.181043:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049521920 : -115096660029696 : ffff9751f876cf00)
00000100:00000001:0.0:1787764357.181051:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764357.181055:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764357.181055:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876cf00 x1874599832716800/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764382 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764357.181064:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000799f2963] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876ed00 x1874599832717056/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764357.181067:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764357.181074:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764357.181078:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d327f8.
00000100:00000001:0.0:1787764357.181080:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000200:3.0:1787764357.181081:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832716800, offset 0
00000100:00000040:0.0:1787764357.181085:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (22081)
00000400:00000200:3.0:1787764357.181089:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764357.181093:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764357.181097:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764357.181100:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764357.181103:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764357.181104:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764357.181106:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764357.181108:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58835:58835:255:4294967295] 192.168.203.110@tcp LPNI seq info [58835:58835:7:4294967295]
00000400:00000200:3.0:1787764357.181111:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764357.181117:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764357.181122:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764357.181125:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186d00.
00000800:00000200:3.0:1787764357.181129:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764357.181163:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787764357.181167:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764357.181171:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:3.0:1787764357.181174:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764357.181177:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316744224
00000100:00000001:3.0:1787764357.181180:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316745459
00000100:00000001:3.0:1787764357.181182:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316744224
00000100:00000001:3.0:1787764357.181185:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316745459
00000100:00000001:3.0:1787764357.181187:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764357.181189:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764357.181191:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764357.181195:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e940 x1874599832716928/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764357.181203:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764357.181205:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764357.181207:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764357.181209:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764357.181211:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764357.181214:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832716928:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764357.181217:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764357.181219:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764357.181221:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764357.181223:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764357.181226:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764357.181228:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764357.181231:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764357.181234:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc234268.
00000400:00000010:3.0:1787764357.181238:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32550.
00000100:00000200:3.0:1787764357.181241:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832716928, portal 10
00000100:00000001:3.0:1787764357.181243:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764357.181244:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049528640 : -115096660022976 : ffff9751f876e940)
00000100:00000040:3.0:1787764357.181250:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e940 x1874599832716928/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764382 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764357.181257:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764357.181259:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764357.181262:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32330.
00000100:00000200:3.0:1787764357.181264:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832716928, offset 0
00000400:00000200:3.0:1787764357.181269:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764357.181272:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764357.181276:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764357.181279:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764357.181282:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764357.181285:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764357.181293:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764357.181296:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58836:58836:254:4294967295] 192.168.203.110@tcp LPNI seq info [58836:58836:6:4294967295]
00000400:00000200:3.0:1787764357.181300:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764357.181306:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764357.181311:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764357.181315:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186700.
00000800:00000200:3.0:1787764357.181319:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764357.181323:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787764357.181326:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764357.181329:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:3.0:1787764357.181332:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764357.181333:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316744212
00000100:00000001:3.0:1787764357.181336:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316745459
00000100:00000001:3.0:1787764357.181338:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316744212
00000100:00000001:3.0:1787764357.181340:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316745459
00000100:00000001:3.0:1787764357.181342:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764357.181344:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764357.181347:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764357.181349:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764357.181353:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764357.181355:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1787764357.181356:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764357.181358:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764357.181360:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:3.0:1787764357.181364:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876ed00 x1874599832717056/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764357.181370:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764357.181372:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764357.181374:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:3.0:1787764357.181376:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:3.0:1787764357.181377:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764357.181381:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832717056:192.168.203.110@tcp:8:kworker.0
00000100:00000001:3.0:1787764357.181384:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:3.0:1787764357.181386:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:3.0:1787764357.181388:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:3.0:1787764357.181389:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:3.0:1787764357.181392:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f8610c00.
02000000:00000001:3.0:1787764357.181394:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:3.0:1787764357.181398:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:3.0:1787764357.181400:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750cc234c60.
00000400:00000010:3.0:1787764357.181404:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d325d8.
00000100:00000200:3.0:1787764357.181407:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832717056, portal 4
00000100:00000001:3.0:1787764357.181409:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:3.0:1787764357.181410:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049529600 : -115096660022016 : ffff9751f876ed00)
00000100:00000040:3.0:1787764357.181416:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876ed00 x1874599832717056/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764382 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764357.181423:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:3.0:1787764357.181425:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:3.0:1787764357.181427:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751f5d32770.
00000100:00000200:3.0:1787764357.181429:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832717056, offset 0
00000400:00000200:3.0:1787764357.181434:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:3.0:1787764357.181438:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764357.181441:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:3.0:1787764357.181444:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1787764357.181447:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:3.0:1787764357.181449:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1787764357.181451:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:3.0:1787764357.181454:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58837:58837:253:4294967295] 192.168.203.110@tcp LPNI seq info [58837:58837:5:4294967295]
00000400:00000200:3.0:1787764357.181458:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:3.0:1787764357.181463:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1787764357.181468:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:3.0:1787764357.181471:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f2186900.
00000800:00000200:3.0:1787764357.181474:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:3.1:1787764357.181478:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:3.0:1787764357.181481:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764357.181483:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:3.0:1787764357.181486:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:3.0:1787764357.181488:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316744229
00000100:00000001:3.0:1787764357.181490:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316745459
00000100:00000001:3.0:1787764357.181493:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316744229
00000100:00000001:3.0:1787764357.181495:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316745459
00000100:00000001:3.0:1787764357.181496:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:3.0:1787764357.181499:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764357.181502:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764357.181504:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764357.181507:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764357.181508:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5)
00000100:00000001:3.0:1787764357.181510:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764357.181511:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764357.181513:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764357.181514:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764357.820219:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764357.820220:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764357.820226:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764357.820227:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764357.820231:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764357.820234:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00100000:2.0:1787764357.820237:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764357.820239:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00000100:00000001:2.0:1787764357.820243:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764357.820244:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00000100:00000001:2.0:1787764357.820247:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764357.820249:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764357.820252:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764357.820255:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00000100:00000001:2.0:1787764357.820256:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764357.820259:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764357.820261:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764357.820263:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764357.820268:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764357.820272:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764357.820274:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764357.820333:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764357.820337:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764357.820340:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764358.204253:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764358.204259:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764358.204261:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764358.204264:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764358.204267:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764358.204270:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764358.204279:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764358.204283:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764358.204286:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764358.460236:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316745459, at 4316745472
00000100:00000010:3.0:1787764358.460250:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40120.
00000100:00000040:2.0:1787764358.460268:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316745459, at 4316745472
00000100:00000010:2.0:1787764358.460276:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e6d40de0.
00000100:00000001:2.0:1787764358.844248:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764358.844256:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764358.844256:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764358.844258:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764358.844261:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764358.844264:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764358.844268:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764358.844270:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764358.844271:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764358.844274:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764358.844275:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764358.844278:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764358.844281:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764358.844282:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764358.844285:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:2.0:1787764358.844308:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:2.0:1787764358.844311:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:2.0:1787764358.844313:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764359.228240:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764359.228246:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764359.228248:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764359.228252:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764359.228255:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764359.228259:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764359.228262:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764359.228263:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764359.228266:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764359.868213:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764359.868220:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764359.868220:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764359.868222:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764359.868225:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764359.868227:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764359.868231:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764359.868231:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764359.868235:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764359.868235:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764359.868240:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764359.868242:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764359.868242:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764359.868245:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764359.868246:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764359.868278:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764359.868281:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764359.868284:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764360.252236:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764360.252244:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764360.252247:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764360.252252:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764360.252256:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764360.252259:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764360.252262:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764360.252265:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764360.252268:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000400:00000100:0.0:1787764360.252436:0:257563:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.110:988
00000400:00000100:0.0:1787764360.252480:0:257563:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.110@tcp at host 192.168.203.110 was unreachable: the network or that node may be down, or Lustre may be misconfigured.
00000400:00000200:0.0:1787764360.252489:0:257563:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764360.252493:0:257563:0:(router.c:1791:lnet_notify()) 192.168.203.10@tcp notifying 192.168.203.110@tcp: down
00000400:00000200:0.0:1787764360.252497:0:257563:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764360.252501:0:257563:0:(router.c:1888:lnet_notify()) Old news
00000800:00000100:0.0:1787764360.252506:0:257563:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000800:00000010:0.0:1787764360.252517:0:257563:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186a00.
00000400:00000200:0.0:1787764360.252522:0:257563:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1787764360.252526:0:257563:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1787764360.252532:0:257563:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32660
00000400:00000010:0.0:1787764360.252534:0:257563:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32660.
00000100:00000001:0.0:1787764360.252540:0:257563:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787764360.252547:0:257563:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f876f840 x1874599832716672/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764382 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764360.252563:0:257563:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787764360.252565:0:257563:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f840 x1874599832716672/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764382 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764360.252572:0:257563:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764360.252574:0:257563:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764360.252595:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000800:00000100:0.0:1787764360.252597:0:257563:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000100:00000001:3.0:1787764360.252600:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000800:00000010:0.0:1787764360.252601:0:257563:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186d00.
00000100:00000001:3.0:1787764360.252605:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000400:00000200:0.0:1787764360.252606:0:257563:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1787764360.252611:0:257563:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1787764360.252616:0:257563:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d327f8
00000100:00100000:3.0:1787764360.252617:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764357/real 1787764360]  req@ffff9751f876f840 x1874599832716672/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1787764360.252619:0:257563:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d327f8.
00000100:00000001:0.0:1787764360.252636:0:257563:0:(events.c:34:request_out_callback()) Process entered
00000400:00000001:3.0:1787764360.252638:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764360.252641:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000100:00000200:0.0:1787764360.252643:0:257563:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f876cf00 x1874599832716800/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764382 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1787764360.252646:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc2342c0.
00000400:00000200:3.0:1787764360.252652:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d326e8
00000400:00000010:3.0:1787764360.252656:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d326e8.
00000100:00000001:3.0:1787764360.252659:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000001:0.0:1787764360.252664:0:257563:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000200:3.0:1787764360.252665:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f840 x1874599832716672/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764360.252670:0:257563:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876cf00 x1874599832716800/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764382 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764360.252686:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f840 x1874599832716672/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764360.252687:0:257563:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764360.252691:0:257563:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:3.0:1787764360.252698:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000800:00000100:0.0:1787764360.252701:0:257563:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000100:00000001:3.0:1787764360.252701:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000800:00000010:0.0:1787764360.252707:0:257563:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186700.
00000100:00000001:3.0:1787764360.252707:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764360.252710:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000400:00000200:0.0:1787764360.252712:0:257563:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000100:00100000:3.0:1787764360.252717:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f840 x1874599832716672/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1787764360.252721:0:257563:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1787764360.252728:0:257563:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32330
00000100:00000001:3.0:1787764360.252730:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000400:00000010:0.0:1787764360.252731:0:257563:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32330.
00000100:00000001:0.0:1787764360.252746:0:257563:0:(events.c:34:request_out_callback()) Process entered
00000100:00000040:3.0:1787764360.252751:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f840 x1874599832716672/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1787764360.252753:0:257563:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f876e940 x1874599832716928/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764382 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764360.252773:0:257563:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787764360.252776:0:257563:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e940 x1874599832716928/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764382 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764360.252780:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764360.252784:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764360.252785:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764360.252786:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764360.252793:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764360.252796:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:0.0:1787764360.252799:0:257563:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:3.0:1787764360.252801:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00000001:0.0:1787764360.252802:0:257563:0:(events.c:68:request_out_callback()) Process leaving
00000100:00080000:3.0:1787764360.252809:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000800:00000100:0.0:1787764360.252812:0:257563:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.110@tcp
00000100:00000001:3.0:1787764360.252814:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764360.252821:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f840 x1874599832716672/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764360.252828:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832716672:192.168.203.110@tcp:250:kworker.0
00000100:00000001:3.0:1787764360.252832:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764360.252834:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764360.252837:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f840 x1874599832716672/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764360.252842:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764360.252845:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764360.252848:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8611c00.
02000000:00000001:3.0:1787764360.252852:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764360.252854:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764360.252855:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:3.0:1787764360.252857:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764360.252859:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015400.
02000000:00000001:3.0:1787764360.252863:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764360.252864:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764360.252866:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f840.
00000100:00000001:3.0:1787764360.252875:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764360.252876:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764360.252879:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764360.252881:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764360.252882:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764360.252883:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764360.252887:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764357/real 1787764360]  req@ffff9751f876cf00 x1874599832716800/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764360.252895:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764360.252897:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764360.252899:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc234f20.
00000400:00000200:3.0:1787764360.252901:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32e58
00000400:00000010:3.0:1787764360.252903:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32e58.
00000100:00000001:3.0:1787764360.252905:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764360.252908:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876cf00 x1874599832716800/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764360.252916:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876cf00 x1874599832716800/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764360.252922:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764360.252923:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764360.252925:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764360.252926:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764360.252929:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876cf00 x1874599832716800/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764360.252936:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764360.252939:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876cf00 x1874599832716800/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764360.252945:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764360.252947:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764360.252948:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764360.252950:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764360.252951:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764360.252963:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764360.252966:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764360.252969:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764360.252971:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764360.252975:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876cf00 x1874599832716800/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764360.252989:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832716800:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764360.252993:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:3.0:1787764360.252997:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764357/real 1787764360]  req@ffff9751f876e940 x1874599832716928/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:3.0:1787764360.253085:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:3.0:1787764360.253086:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:3.0:1787764360.253088:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc234268.
00000400:00000200:3.0:1787764360.253090:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32550
00000400:00000010:3.0:1787764360.253092:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32550.
00000100:00000001:3.0:1787764360.253094:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:3.0:1787764360.253098:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e940 x1874599832716928/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1787764360.253105:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e940 x1874599832716928/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764360.253111:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:3.0:1787764360.253112:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764360.253114:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764360.253115:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:3.0:1787764360.253118:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e940 x1874599832716928/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764360.253123:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:3.0:1787764360.253126:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e940 x1874599832716928/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764360.253132:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:3.0:1787764360.253134:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764360.253153:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:3.0:1787764360.253155:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764360.253158:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:3.0:1787764360.253160:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:3.0:1787764360.253163:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:3.0:1787764360.253167:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:3.0:1787764360.253170:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:3.0:1787764360.253174:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e940 x1874599832716928/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:3.0:1787764360.253180:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832716928:192.168.203.110@tcp:38:kworker.0
00000100:00000001:3.0:1787764360.253184:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764360.253186:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764360.253189:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876cf00 x1874599832716800/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764360.253195:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764360.253196:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764360.253197:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8612000.
02000000:00000001:3.0:1787764360.253200:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764360.253201:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764360.253203:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764360.253205:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764360.253206:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016000.
02000000:00000001:3.0:1787764360.253209:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764360.253211:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764360.253213:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876cf00.
00000100:00000001:3.0:1787764360.253216:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764360.253218:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764360.253220:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:3.0:1787764360.253223:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e940 x1874599832716928/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:3.0:1787764360.253230:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:3.0:1787764360.253232:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:3.0:1787764360.253233:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610800.
02000000:00000001:3.0:1787764360.253236:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:3.0:1787764360.253237:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:3.0:1787764360.253238:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:3.0:1787764360.253239:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:3.0:1787764360.253240:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016c00.
02000000:00000001:3.0:1787764360.253242:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:3.0:1787764360.253242:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:3.0:1787764360.253243:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e940.
00000100:00000001:3.0:1787764360.253246:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:3.0:1787764360.253247:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764360.253248:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764360.253250:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:3.0:1787764360.253251:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764360.253253:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:3.0:1787764360.253254:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000800:00000010:0.0:1787764360.253834:0:257563:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f2186900.
00000400:00000200:0.0:1787764360.253841:0:257563:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8
00000400:00000200:0.0:1787764360.253848:0:257563:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_DROPPED
00000400:00000200:0.0:1787764360.253853:0:257563:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d32770
00000400:00000010:0.0:1787764360.253855:0:257563:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d32770.
00000100:00000001:0.0:1787764360.253859:0:257563:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:0.0:1787764360.253865:0:257563:0:(events.c:39:request_out_callback()) @@@ type 5, status -113  req@ffff9751f876ed00 x1874599832717056/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764382 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764360.253884:0:257563:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:0.0:1787764360.253887:0:257563:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876ed00 x1874599832717056/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764382 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764360.253894:0:257563:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764360.253895:0:257563:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764360.253958:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764360.253962:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764360.253974:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764360.253981:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764357/real 1787764360]  req@ffff9751f876ed00 x1874599832717056/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764360.253992:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764360.253994:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764360.253996:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750cc234c60.
00000400:00000200:1.0:1787764360.253999:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751f5d325d8
00000400:00000010:1.0:1787764360.254000:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751f5d325d8.
00000100:00000001:1.0:1787764360.254003:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764360.254006:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876ed00 x1874599832717056/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764360.254015:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876ed00 x1874599832717056/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764360.254022:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764360.254024:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764360.254027:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764360.254029:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764360.254033:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876ed00 x1874599832717056/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764360.254041:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764360.254045:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876ed00 x1874599832717056/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764360.254052:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764360.254055:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764360.254057:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764360.254058:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764360.254060:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764360.254062:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764360.254065:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764360.254069:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764360.254072:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764360.254076:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876ed00 x1874599832717056/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764360.254088:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832717056:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764360.254092:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764360.254094:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764360.254097:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876ed00 x1874599832717056/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764382 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764360.254103:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764360.254106:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764360.254108:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f8610c00.
02000000:00000001:1.0:1787764360.254111:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764360.254113:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764360.254115:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787764360.254116:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764360.254118:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017800.
02000000:00000001:1.0:1787764360.254122:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764360.254123:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764360.254124:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876ed00.
00000100:00000001:1.0:1787764360.254127:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764360.254128:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764360.254130:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764360.254163:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764360.254600:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764360.254606:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764360.254609:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764360.892260:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764360.892266:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764360.892268:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764360.892273:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764360.892273:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764360.892283:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764360.892284:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764360.892289:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764360.892292:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764360.892293:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764360.892298:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764360.892299:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764360.892307:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764360.892309:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764360.892312:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764360.892377:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764360.892381:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764360.892386:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764361.276226:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764361.276235:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764361.276237:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764361.276242:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764361.276245:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764361.276249:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764361.276251:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764361.276254:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764361.276257:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764361.916217:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764361.916229:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764361.916231:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764361.916241:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764361.916241:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764361.916249:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764361.916252:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764361.916256:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764361.916258:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764361.916260:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764361.916263:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764361.916267:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764361.916269:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764361.916272:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764361.916275:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764361.916330:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764361.916333:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764361.916335:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764362.300259:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764362.300266:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764362.300268:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764362.300271:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764362.300274:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764362.300277:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764362.300279:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764362.300280:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764362.300283:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764362.300284:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 62ab3367-239a-408f-ae8e-abd2add05f2e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764362.300290:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery
00000100:00000001:0.0:1787764362.300292:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764362.300295:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764362.300298:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764362.300304:0:279196:0:(import.c:510:import_select_connection()) MGC192.168.203.110@tcp: connect to NID 192.168.203.110@tcp last attempt 22076
00000400:00000200:0.0:1787764362.300313:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764362.300321:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764362.300325:0:279196:0:(import.c:589:import_select_connection()) MGC192.168.203.110@tcp: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764362.300332:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764362.300335:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764362.300341:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764362.300344:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764362.300350:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764362.300353:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375417
00000020:00000001:0.0:1787764362.300357:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764362.300360:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751e0c8f800 refcount=3
00000020:00000001:0.0:1787764362.300362:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000020:00000001:0.0:1787764362.300365:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976652253184 : -115097057298432 : ffff9751e0c8f800)
00000100:00000040:0.0:1787764362.300367:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764362.300369:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764362.300370:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764362.300371:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764362.300373:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751e0c8f800 : new refcount 2
00000100:00080000:0.0:1787764362.300376:0:279196:0:(import.c:624:import_select_connection()) MGC192.168.203.110@tcp: import ffff9750e2847000 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764362.300377:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764362.300379:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:0.0:1787764362.300381:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:0.0:1787764362.300383:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764362.300384:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000100:00000001:0.0:1787764362.300386:0:279196:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764362.300393:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876f0c0.
00000020:00000040:0.0:1787764362.300395:0:279196:0:(genops.c:1210:class_import_get()) import ffff9750e2847000 refcount=4 obd=MGC192.168.203.110@tcp
02000000:00000001:0.0:1787764362.300398:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764362.300399:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764362.300401:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764362.300404:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764362.300408:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:0.0:1787764362.300416:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9014400.
00000100:00000001:0.0:1787764362.300421:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764362.300430:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876f0c0 x1874599832717184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:0.0:1787764362.300440:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764362.300443:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764362.300448:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ed7658] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876f0c0 x1874599832717184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764362.300468:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764362.300471:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00080000:0.0:1787764362.300474:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery
00000100:00000001:0.0:1787764362.300476:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764362.300477:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764362.300480:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00080000:0.0:1787764362.300482:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22076
00000400:00000200:0.0:1787764362.300487:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764362.300490:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00080000:0.0:1787764362.300492:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764362.300496:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764362.300503:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000001:1.0:1787764362.300505:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000040:0.0:1787764362.300506:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764362.300511:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000100:00000001:1.0:1787764362.300511:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764362.300515:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000020:00000001:0.0:1787764362.300516:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764362.300520:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d937546b
00000020:00000001:0.0:1787764362.300522:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764362.300526:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12a000 refcount=5
00000100:00000040:1.0:1787764362.300527:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876f0c0 x1874599832717184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787764362.300531:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000020:00000001:0.0:1787764362.300537:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513107968 : -115101196443648 : ffff9750ea12a000)
00000100:00000040:0.0:1787764362.300544:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:1.0:1787764362.300546:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764362.300549:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764362.300553:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
00000100:00000001:0.0:1787764362.300554:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
02000000:00000001:1.0:1787764362.300557:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
00000100:00000040:0.0:1787764362.300558:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
02000000:00000001:1.0:1787764362.300564:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764362.300565:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764362.300570:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12a000 : new refcount 4
00000100:00100000:1.0:1787764362.300572:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832717184:192.168.203.110@tcp:250:kworker.0
00000100:00080000:0.0:1787764362.300576:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9750da7c7000: import ffff9751d938e800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764362.300580:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
00000100:00000001:0.0:1787764362.300581:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764362.300585:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
02000000:00000001:1.0:1787764362.300586:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764362.300590:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764362.300591:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764362.300593:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000001:0.0:1787764362.300595:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
02000000:00000010:1.0:1787764362.300601:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cac00.
00000100:00000001:0.0:1787764362.300605:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
02000000:00000001:1.0:1787764362.300606:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000008:00100000:0.0:1787764362.300610:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000200:1.0:1787764362.300615:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000008:00000001:0.0:1787764362.300617:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764362.300621:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787764362.300622:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000100:00000010:0.0:1787764362.300625:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876c000.
00000020:00000040:0.0:1787764362.300629:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938e800 refcount=65 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000400:00000010:1.0:1787764362.300632:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
02000000:00000001:0.0:1787764362.300633:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764362.300635:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764362.300638:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764362.300639:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
00000100:00000200:1.0:1787764362.300641:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832717184, portal 25
02000000:00000001:0.0:1787764362.300642:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764362.300647:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
02000000:00000010:0.0:1787764362.300650:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9017000.
00000100:00000001:1.0:1787764362.300651:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049530560 : -115096660021056 : ffff9751f876f0c0)
00000100:00000001:0.0:1787764362.300657:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764362.300661:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876f0c0 x1874599832717184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:0.0:1787764362.300667:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876c000 x1874599832717312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764362.300677:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000001:0.0:1787764362.300683:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000040:1.0:1787764362.300684:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787764362.300688:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000010:1.0:1787764362.300689:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000040:0.0:1787764362.300695:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003dcba473] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876c000 x1874599832717312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764362.300695:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874599832717184, offset 0
00000400:00000200:1.0:1787764362.300707:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787764362.300710:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00080000:0.0:1787764362.300714:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000400:00000200:1.0:1787764362.300718:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00080000:0.0:1787764362.300720:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery
00000100:00000001:0.0:1787764362.300734:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764362.300735:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:0.0:1787764362.300740:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000400:00000200:1.0:1787764362.300740:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000100:00080000:0.0:1787764362.300746:0:279196:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22076
00000400:00000200:1.0:1787764362.300749:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764362.300756:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764362.300759:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764362.300762:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764362.300766:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58838:58838:256:4294967295] 192.168.203.110@tcp LPNI seq info [58838:58838:8:4294967295]
00000400:00000200:1.0:1787764362.300770:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764362.300776:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:0.0:1787764362.300783:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000400:00000200:1.0:1787764362.300786:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000100:00080000:0.0:1787764362.300788:0:279196:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764362.300795:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:1.0:1787764362.300799:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000100:00000001:0.0:1787764362.300800:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000040:0.0:1787764362.300805:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764362.300808:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000800:00000010:1.0:1787764362.300810:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000020:00000001:0.0:1787764362.300812:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000020:00000040:0.0:1787764362.300815:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375472
00000020:00000001:0.0:1787764362.300817:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000020:00000040:0.0:1787764362.300820:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9750ea12d800 refcount=5
00000800:00000200:1.0:1787764362.300821:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000020:00000001:0.0:1787764362.300826:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000800:00000200:1.0:1787764362.300827:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 57 secs later)
00000020:00000001:0.0:1787764362.300830:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628972513122304 : -115101196429312 : ffff9750ea12d800)
00000100:00000040:0.0:1787764362.300838:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000800:00000200:1.1:1787764362.300838:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000100:00000001:0.0:1787764362.300842:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000800:00000200:1.1:1787764362.300845:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 57 secs later)
00000100:00000040:0.0:1787764362.300846:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764362.300851:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764362.300856:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9750ea12d800 : new refcount 4
00000800:00000100:1.0:1787764362.300856:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764362.300861:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000100:00080000:0.0:1787764362.300863:0:279196:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9750da7c7000: import ffff9751d938b800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:0.0:1787764362.300867:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:0.0:1787764362.300869:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000400:00000200:1.0:1787764362.300869:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
02000000:00000001:0.0:1787764362.300874:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000400:00000200:1.0:1787764362.300878:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
02000000:00000001:0.0:1787764362.300879:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764362.300884:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000400:00000200:1.0:1787764362.300885:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000008:00100000:0.0:1787764362.300886:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683198 ocd_grant: 2097152, lost: 0.
00000400:00000010:1.0:1787764362.300889:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000008:00000001:0.0:1787764362.300891:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764362.300895:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764362.300895:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000010:0.0:1787764362.300900:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e1c0.
00000100:00000200:1.0:1787764362.300902:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876f0c0 x1874599832717184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787764362.300903:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751d938b800 refcount=4 obd=lustre-MDT0001-mdc-ffff9750da7c7000
02000000:00000001:0.0:1787764362.300908:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764362.300911:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764362.300915:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
02000000:00000001:0.0:1787764362.300919:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764362.300922:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764362.300922:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
02000000:00000010:0.0:1787764362.300927:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9015800.
00000100:00000040:1.0:1787764362.300928:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876f0c0 x1874599832717184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 0 dl 1787764387 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764362.300932:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764362.300941:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:0.0:1787764362.300942:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e1c0 x1874599832717440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00000001:1.0:1787764362.300944:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:0.0:1787764362.300952:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:1.0:1787764362.300952:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764362.300957:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764362.300960:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000040:0.0:1787764362.300965:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000050d81244] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e1c0 x1874599832717440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.300966:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 2 is 4316750579
00000100:00000010:1.0:1787764362.300974:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4d80.
00000100:00000001:0.0:1787764362.300976:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764362.300980:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787764362.300981:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 2
00000100:00000040:0.0:1787764362.300986:0:279196:0:(pinger.c:247:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE)
00000100:00080000:0.0:1787764362.300989:0:279196:0:(pinger.c:229:ptlrpc_pinger_process_import()) 566ea680-1cf8-4b4d-a1e5-37d1741eae1c->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0
00000100:00000001:1.0:1787764362.300991:0:257575:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS new deadline estimation for cpu 3 is 4316750579
00000100:00080000:0.0:1787764362.300994:0:279196:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery
00000100:00000010:1.0:1787764362.300996:0:257575:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9750e35c4480.
00000100:00000001:0.0:1787764362.300997:0:279196:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered
00000100:00080000:0.0:1787764362.301000:0:279196:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING
00000100:00000001:1.0:1787764362.301002:0:257575:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.110@tcp: PM QoS request now active for cpu 3
00000100:00000001:0.0:1787764362.301005:0:279196:0:(import.c:485:import_select_connection()) Process entered
00000100:00000001:1.0:1787764362.301006:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00080000:0.0:1787764362.301010:0:279196:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: connect to NID 192.168.203.110@tcp last attempt 22076
00000100:00000001:1.0:1787764362.301012:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:0.0:1787764362.301016:0:279196:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764362.301017:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000400:00000200:0.0:1787764362.301023:0:279196:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.203.110@tcp is discovered: rc = 1
00000100:00100000:1.0:1787764362.301025:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764362/real 1787764362]  req@ffff9751f876f0c0 x1874599832717184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00080000:0.0:1787764362.301027:0:279196:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: tried all connections, increasing latency to 20s
00000100:00000040:0.0:1787764362.301035:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764362.301041:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000400:00000001:1.0:1787764362.301042:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000100:00000040:0.0:1787764362.301044:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000400:00000001:1.0:1787764362.301045:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764362.301049:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000100:00000001:0.0:1787764362.301050:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000001:0.0:1787764362.301054:0:279196:0:(genops.c:792:class_conn2export()) Process entered
00000400:00000200:1.0:1787764362.301054:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000020:00000040:0.0:1787764362.301057:0:279196:0:(genops.c:804:class_conn2export()) looking for export cookie 0xf0901072d9375487
00000400:00000010:1.0:1787764362.301059:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000020:00000001:0.0:1787764362.301061:0:279196:0:(lustre_handles.c:130:class_handle2object()) Process entered
00000100:00000001:1.0:1787764362.301063:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000020:00000040:0.0:1787764362.301065:0:279196:0:(lustre_handles.c:148:class_handle2object()) GET export ffff9751c866d000 refcount=4
00000020:00000001:0.0:1787764362.301070:0:279196:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000200:1.0:1787764362.301070:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876f0c0 x1874599832717184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000001:0.0:1787764362.301074:0:279196:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446628976243167232 : -115097466384384 : ffff9751c866d000)
00000100:00000040:0.0:1787764362.301082:0:279196:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff9751c6c3a340 refcount 14 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764362.301085:0:279196:0:(connection.c:156:ptlrpc_connection_addref()) Process entered
00000100:00000200:1.0:1787764362.301088:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876f0c0 x1874599832717184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764362.301090:0:279196:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9751c6c3a340 refcount 15 to 192.168.203.110@tcp
00000100:00000001:0.0:1787764362.301095:0:279196:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446628976215696192 : -115097493855424 : ffff9751c6c3a340)
00000020:00000040:0.0:1787764362.301099:0:279196:0:(genops.c:879:class_export_put()) PUTting export ffff9751c866d000 : new refcount 3
00000100:00080000:0.0:1787764362.301105:0:279196:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9750da7c7000: import ffff9751e0c5d800 using connection 192.168.203.110@tcp/192.168.203.110@tcp
00000100:00000001:1.0:1787764362.301106:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:0.0:1787764362.301108:0:279196:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764362.301109:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787764362.301111:0:279196:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered
00000100:00000001:1.0:1787764362.301113:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000001:0.0:1787764362.301116:0:279196:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764362.301117:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
02000000:00000001:0.0:1787764362.301122:0:279196:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764362.301125:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876f0c0 x1874599832717184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764362.301126:0:279196:0:(obd_class.h:888:obd_reconnect()) Process entered
00000008:00100000:0.0:1787764362.301129:0:279196:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683198 ocd_grant: 8388608, lost: 0.
00000008:00000001:0.0:1787764362.301165:0:279196:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764362.301168:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764362.301169:0:279196:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0)
00000100:00000010:0.0:1787764362.301177:0:279196:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 904 at ffff9751f876e580.
00000100:00000040:1.0:1787764362.301179:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876f0c0 x1874599832717184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000020:00000040:0.0:1787764362.301180:0:279196:0:(genops.c:1210:class_import_get()) import ffff9751e0c5d800 refcount=3 obd=lustre-OST0001-osc-ffff9750da7c7000
02000000:00000001:0.0:1787764362.301184:0:279196:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered
02000000:00000001:0.0:1787764362.301187:0:279196:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:0.0:1787764362.301191:0:279196:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered
00000100:00000001:1.0:1787764362.301193:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
02000000:00000001:0.0:1787764362.301194:0:279196:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered
02000000:00000001:0.0:1787764362.301198:0:279196:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764362.301199:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764362.301203:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
02000000:00000010:0.0:1787764362.301204:0:279196:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9750e9016800.
00000100:00000001:1.0:1787764362.301205:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:0.0:1787764362.301210:0:279196:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764362.301210:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764362.301213:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764362.301218:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9750e2847000 MGS: changing import state from CONNECTING to DISCONN
00000100:00100000:0.0:1787764362.301220:0:279196:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5)  req@ffff9751f876e580 x1874599832717568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295
00000100:00080000:1.0:1787764362.301226:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764362.301231:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000001:0.0:1787764362.301235:0:279196:0:(jobid.c:939:lustre_get_jobid()) Process entered
00000100:00000001:0.0:1787764362.301239:0:279196:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764362.301240:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876f0c0 x1874599832717184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000040:0.0:1787764362.301246:0:279196:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dd5b856c] to pc [ptlrpcd_rcv+-1]  req@ffff9751f876e580 x1874599832717568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:0.0:1787764362.301262:0:279196:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:0.0:1787764362.301266:0:279196:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (22087)
00000100:00100000:1.0:1787764362.301274:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:62ab3367-239a-408f-ae8e-abd2add05f2e:257575:1874599832717184:192.168.203.110@tcp:250:kworker.0
00000100:00000001:1.0:1787764362.301282:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764362.301284:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764362.301288:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876f0c0 x1874599832717184/t0(0) o250->MGC192.168.203.110@tcp@192.168.203.110@tcp:26/25 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.301295:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764362.301297:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764362.301299:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787764362.301302:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764362.301304:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764362.301306:0:257575:0:(genops.c:1223:class_import_put()) import ffff9750e2847000 refcount=3 obd=MGC192.168.203.110@tcp
00000020:00000001:1.0:1787764362.301308:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764362.301310:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9014400.
02000000:00000001:1.0:1787764362.301313:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764362.301315:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764362.301317:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876f0c0.
00000100:00000001:1.0:1787764362.301320:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764362.301321:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764362.301323:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764362.301330:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764362.301331:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764362.301333:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764362.301335:0:257575:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:1.0:1787764362.301336:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764362.301340:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876c000 x1874599832717312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.301352:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764362.301354:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764362.301356:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764362.301357:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764362.301359:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764362.301362:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832717312:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764362.301366:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764362.301368:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764362.301369:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764362.301371:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764362.301374:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787764362.301376:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764362.301380:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764362.301383:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764362.301387:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764362.301390:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832717312, portal 10
00000100:00000001:1.0:1787764362.301392:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764362.301393:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049518080 : -115096660033536 : ffff9751f876c000)
00000100:00000040:1.0:1787764362.301399:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876c000 x1874599832717312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.302651:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764362.302656:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764362.302661:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764362.302666:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832717312, offset 0
00000400:00000200:1.0:1787764362.302670:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764362.302678:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764362.302683:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764362.302687:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764362.302690:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764362.302694:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764362.302696:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764362.302700:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58839:58839:256:4294967295] 192.168.203.110@tcp LPNI seq info [58839:58839:8:4294967295]
00000400:00000200:1.0:1787764362.302704:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764362.302712:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764362.302718:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764362.302721:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764362.302736:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764362.302738:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:1.1:1787764362.302742:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764362.302744:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 57 secs later)
00000800:00000100:1.0:1787764362.302746:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764362.302748:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764362.302751:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764362.302754:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764362.302757:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764362.302759:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764362.302761:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764362.302765:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876c000 x1874599832717312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.302773:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764362.302775:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876c000 x1874599832717312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764387 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.302779:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764362.302780:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764362.302784:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764362.302788:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: using a timeout of 30926 usecs (31 jiffies)
00000400:00000200:1.0:1787764362.302790:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764362.302791:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316749345
00000100:00000001:1.0:1787764362.302793:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316750579
00000100:00000001:1.0:1787764362.302795:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316749345
00000100:00000001:1.0:1787764362.302796:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316750579
00000100:00000001:1.0:1787764362.302798:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764362.302800:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764362.302801:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764362.302804:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764362/real 1787764362]  req@ffff9751f876c000 x1874599832717312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764362.302809:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764362.302810:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764362.302812:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764362.302813:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764362.302814:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764362.302815:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764362.302817:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876c000 x1874599832717312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764362.302822:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876c000 x1874599832717312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.302825:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764362.302826:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764362.302827:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764362.302828:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764362.302830:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876c000 x1874599832717312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.302834:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764362.302836:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876c000 x1874599832717312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.302841:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764362.302843:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764362.302843:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764362.302844:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764362.302846:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764362.302855:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764362.302857:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764362.302860:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764362.302861:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764362.302863:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876c000 x1874599832717312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764362.302868:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832717312:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764362.302870:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764362.302873:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e1c0 x1874599832717440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.302876:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764362.302877:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764362.302879:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764362.302880:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764362.302882:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764362.302883:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832717440:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764362.302886:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764362.302888:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764362.302889:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764362.302891:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764362.302893:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787764362.302895:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764362.302897:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764362.302898:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764362.302900:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764362.302902:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832717440, portal 10
00000100:00000001:1.0:1787764362.302904:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764362.302905:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049526720 : -115096660024896 : ffff9751f876e1c0)
00000100:00000040:1.0:1787764362.302911:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e1c0 x1874599832717440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.302917:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764362.302919:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764362.302921:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764362.302923:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874599832717440, offset 0
00000400:00000200:1.0:1787764362.302926:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764362.302929:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764362.302933:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764362.302936:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764362.302938:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764362.302940:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764362.302942:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764362.302945:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58840:58840:256:4294967295] 192.168.203.110@tcp LPNI seq info [58840:58840:8:4294967295]
00000400:00000200:1.0:1787764362.302949:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764362.302956:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764362.302960:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764362.302963:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764362.302966:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764362.302968:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:1.1:1787764362.302971:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764362.302972:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 57 secs later)
00000800:00000100:1.0:1787764362.302974:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764362.302976:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764362.302977:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764362.302979:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764362.302981:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764362.302982:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764362.302984:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764362.302986:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e1c0 x1874599832717440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.302991:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764362.302993:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e1c0 x1874599832717440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 0 dl 1787764387 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.302997:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764362.302998:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764362.303000:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764362.303002:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: using a timeout of 18491 usecs (19 jiffies)
00000400:00000200:1.0:1787764362.303004:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764362.303005:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316749333
00000100:00000001:1.0:1787764362.303007:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316750579
00000100:00000001:1.0:1787764362.303008:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316749333
00000100:00000001:1.0:1787764362.303009:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316750579
00000100:00000001:1.0:1787764362.303010:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764362.303012:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764362.303013:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764362.303015:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764362/real 1787764362]  req@ffff9751f876e1c0 x1874599832717440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764362.303019:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764362.303020:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764362.303021:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764362.303022:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764362.303023:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764362.303025:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764362.303027:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e1c0 x1874599832717440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764362.303031:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e1c0 x1874599832717440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.303035:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764362.303036:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764362.303038:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764362.303039:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764362.303042:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e1c0 x1874599832717440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.303048:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764362.303053:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e1c0 x1874599832717440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.303059:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764362.303061:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764362.303062:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764362.303064:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764362.303065:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764362.303067:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764362.303069:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751d938b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764362.303072:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764362.303074:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764362.303077:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e1c0 x1874599832717440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764362.303085:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832717440:192.168.203.110@tcp:38:kworker.0
00000100:00000001:1.0:1787764362.303089:0:257575:0:(client.c:1813:ptlrpc_send_new_req()) Process entered
00000100:00000040:1.0:1787764362.303091:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc  req@ffff9751f876e580 x1874599832717568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.303095:0:257575:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:1.0:1787764362.303096:0:257575:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764362.303097:0:257575:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered
02000000:00000001:1.0:1787764362.303098:0:257575:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0)
02000000:00000001:1.0:1787764362.303100:0:257575:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:1.0:1787764362.303101:0:257575:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832717568:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764362.303103:0:257575:0:(niobuf.c:866:ptl_send_rpc()) Process entered
02000000:00000001:1.0:1787764362.303105:0:257575:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered
02000000:00000001:1.0:1787764362.303106:0:257575:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0)
02000000:00000001:1.0:1787764362.303107:0:257575:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered
02000000:00000010:1.0:1787764362.303109:0:257575:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787764362.303110:0:257575:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0)
00000400:00000200:1.0:1787764362.303113:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000010:1.0:1787764362.303114:0:257575:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9750e9210318.
00000400:00000010:1.0:1787764362.303116:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c7933dd0.
00000100:00000200:1.0:1787764362.303119:0:257575:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874599832717568, portal 4
00000100:00000001:1.0:1787764362.303121:0:257575:0:(client.c:3275:ptlrpc_request_addref()) Process entered
00000100:00000001:1.0:1787764362.303122:0:257575:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446628977049527680 : -115096660023936 : ffff9751f876e580)
00000100:00000040:1.0:1787764362.303127:0:257575:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200  req@ffff9751f876e580 x1874599832717568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.303133:0:257575:0:(niobuf.c:69:ptl_send_buf()) Process entered
00000100:00000040:1.0:1787764362.303151:0:257575:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.110@tcp
00000400:00000010:1.0:1787764362.303154:0:257575:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9751c79334c8.
00000100:00000200:1.0:1787764362.303156:0:257575:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874599832717568, offset 0
00000400:00000200:1.0:1787764362.303160:0:257575:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.110@tcp
00000400:00000200:1.0:1787764362.303164:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764362.303167:0:257575:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.110@tcp local destination
00000400:00000200:1.0:1787764362.303170:0:257575:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1787764362.303173:0:257575:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.10@tcp
00000400:00000200:1.0:1787764362.303175:0:257575:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.10@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1787764362.303176:0:257575:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.110@tcp
00000400:00000200:1.0:1787764362.303178:0:257575:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.10@tcp NI seq info: [58841:58841:256:4294967295] 192.168.203.110@tcp LPNI seq info [58841:58841:8:4294967295]
00000400:00000200:1.0:1787764362.303181:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000400:00000200:1.0:1787764362.303185:0:257575:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.10@tcp(192.168.203.10@tcp:<?>) -> 192.168.203.110@tcp(192.168.203.110@tcp:192.168.203.110@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1787764362.303187:0:257575:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764362.303190:0:257575:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9750f66b1600.
00000800:00000200:1.0:1787764362.303193:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.0:1787764362.303196:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 57 secs later)
00000800:00000200:1.1:1787764362.303200:0:257575:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9751d073e480] -> 12345-192.168.203.110@tcp (2)
00000800:00000200:1.1:1787764362.303202:0:257575:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.203.110 (cnted 0, interval 60, 57 secs later)
00000800:00000100:1.0:1787764362.303206:0:257575:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.110@tcp
00000800:00000010:1.0:1787764362.303208:0:257575:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9750f66b1600.
00000400:00000200:1.0:1787764362.303211:0:257575:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7
00000400:00000200:1.0:1787764362.303214:0:257575:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.10@tcp->192.168.203.110@tcp: PUT: REMOTE_ERROR
00000400:00000200:1.0:1787764362.303216:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c79334c8
00000400:00000010:1.0:1787764362.303218:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c79334c8.
00000100:00000001:1.0:1787764362.303221:0:257575:0:(events.c:34:request_out_callback()) Process entered
00000100:00000200:1.0:1787764362.303224:0:257575:0:(events.c:39:request_out_callback()) @@@ type 5, status -5  req@ffff9751f876e580 x1874599832717568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764387 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.303231:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764362.303234:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1  req@ffff9751f876e580 x1874599832717568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 0 dl 1787764387 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.303241:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764362.303242:0:257575:0:(events.c:68:request_out_callback()) Process leaving
00000100:00000001:1.0:1787764362.303245:0:257575:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:1.0:1787764362.303248:0:257575:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: using a timeout of 35430 usecs (36 jiffies)
00000400:00000200:1.0:1787764362.303251:0:257575:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.110@tcp to cpt 1
00000100:00000001:1.0:1787764362.303252:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 2 is 4316749351
00000100:00000001:1.0:1787764362.303254:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 2, keep current deadline 4316750579
00000100:00000001:1.0:1787764362.303255:0:257575:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS new deadline estimation for cpu 3 is 4316749351
00000100:00000001:1.0:1787764362.303256:0:257575:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9750da7c7000: PM QoS request active for cpu 3, keep current deadline 4316750579
00000100:00000001:1.0:1787764362.303258:0:257575:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0)
00000100:00000001:1.0:1787764362.303259:0:257575:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764362.303260:0:257575:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered
00000100:00100000:1.0:1787764362.303262:0:257575:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787764362/real 1787764362]  req@ffff9751f876e580 x1874599832717568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000001:1.0:1787764362.303267:0:257575:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered
00000400:00000001:1.0:1787764362.303267:0:257575:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving
00000400:00000010:1.0:1787764362.303276:0:257575:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9750e9210318.
00000400:00000200:1.0:1787764362.303279:0:257575:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9751c7933dd0
00000400:00000010:1.0:1787764362.303280:0:257575:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9751c7933dd0.
00000100:00000001:1.0:1787764362.303282:0:257575:0:(events.c:78:reply_in_callback()) Process entered
00000100:00000200:1.0:1787764362.303285:0:257575:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9751f876e580 x1874599832717568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1787764362.303294:0:257575:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9751f876e580 x1874599832717568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.303299:0:257575:0:(events.c:163:reply_in_callback()) Process leaving
00000100:00000001:1.0:1787764362.303300:0:257575:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764362.303302:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764362.303303:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00100000:1.0:1787764362.303307:0:257575:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING)  req@ffff9751f876e580 x1874599832717568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.303312:0:257575:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1)
00000100:00000040:1.0:1787764362.303316:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret  req@ffff9751f876e580 x1874599832717568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.303322:0:257575:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00000001:1.0:1787764362.303324:0:257575:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764362.303325:0:257575:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered
00000100:00000001:1.0:1787764362.303326:0:257575:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764362.303329:0:257575:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered
00000100:00000001:1.0:1787764362.303330:0:257575:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92)
00000100:00080000:1.0:1787764362.303332:0:257575:0:(import.c:65:import_set_state_nolock()) ffff9751e0c5d800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN
00000100:00080000:1.0:1787764362.303336:0:257575:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.110@tcp failed (-110)
00000100:00000001:1.0:1787764362.303338:0:257575:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92)
00000100:00000040:1.0:1787764362.303342:0:257575:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete  req@ffff9751f876e580 x1874599832717568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00100000:1.0:1787764362.303346:0:257575:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9751f876e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:566ea680-1cf8-4b4d-a1e5-37d1741eae1c:257575:1874599832717568:192.168.203.110@tcp:8:kworker.0
00000100:00000001:1.0:1787764362.303349:0:257575:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764362.303351:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764362.303354:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876c000 x1874599832717312/t0(0) o38->lustre-MDT0000-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.303359:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764362.303361:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764362.303364:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86cac00.
02000000:00000001:1.0:1787764362.303366:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764362.303369:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764362.303371:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938e800 refcount=64 obd=lustre-MDT0000-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764362.303373:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764362.303375:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9017000.
02000000:00000001:1.0:1787764362.303379:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764362.303381:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764362.303382:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876c000.
00000100:00000001:1.0:1787764362.303385:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764362.303387:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764362.303388:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764362.303392:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e1c0 x1874599832717440/t0(0) o38->lustre-MDT0001-mdc-ffff9750da7c7000@192.168.203.110@tcp:12/10 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.303398:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764362.303399:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764362.303400:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9400.
02000000:00000001:1.0:1787764362.303401:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764362.303402:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764362.303403:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751d938b800 refcount=3 obd=lustre-MDT0001-mdc-ffff9750da7c7000
00000020:00000001:1.0:1787764362.303404:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764362.303405:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9015800.
02000000:00000001:1.0:1787764362.303406:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764362.303406:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764362.303407:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e1c0.
00000100:00000001:1.0:1787764362.303409:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764362.303410:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764362.303411:0:257575:0:(client.c:2867:__ptlrpc_req_put()) Process entered
00000100:00000040:1.0:1787764362.303413:0:257575:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0  req@ffff9751f876e580 x1874599832717568/t0(0) o8->lustre-OST0001-osc-ffff9750da7c7000@192.168.203.110@tcp:28/4 lens 520/544 e 0 to 1 dl 1787764387 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000001:1.0:1787764362.303416:0:257575:0:(client.c:2793:__ptlrpc_free_req()) Process entered
02000000:00000001:1.0:1787764362.303417:0:257575:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered
02000000:00000010:1.0:1787764362.303418:0:257575:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9751f86c9c00.
02000000:00000001:1.0:1787764362.303419:0:257575:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving
00000020:00000001:1.0:1787764362.303419:0:257575:0:(genops.c:1217:class_import_put()) Process entered
00000020:00000040:1.0:1787764362.303420:0:257575:0:(genops.c:1223:class_import_put()) import ffff9751e0c5d800 refcount=2 obd=lustre-OST0001-osc-ffff9750da7c7000
00000020:00000001:1.0:1787764362.303422:0:257575:0:(genops.c:1230:class_import_put()) Process leaving
02000000:00000010:1.0:1787764362.303422:0:257575:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9750e9016800.
02000000:00000001:1.0:1787764362.303424:0:257575:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered
02000000:00000001:1.0:1787764362.303425:0:257575:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving
00000100:00000010:1.0:1787764362.303426:0:257575:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 904 at ffff9751f876e580.
00000100:00000001:1.0:1787764362.303427:0:257575:0:(client.c:2849:__ptlrpc_free_req()) Process leaving
00000100:00000001:1.0:1787764362.303428:0:257575:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764362.303429:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:1.0:1787764362.303433:0:257575:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:1.0:1787764362.303434:0:257575:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:1.0:1787764362.303436:0:257575:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered
00000100:00000001:1.0:1787764362.303437:0:257575:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764362.940226:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764362.940230:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764362.940233:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764362.940235:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764362.940236:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764362.940242:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764362.940244:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764362.940246:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764362.940247:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764362.940249:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764362.940253:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00010000:00000001:3.0:1787764362.940253:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764362.940256:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764362.940257:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764362.940260:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:0.0:1787764362.940289:0:285879:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:0.0:1787764362.940292:0:285879:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:0.0:1787764362.940295:0:285879:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764363.324203:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764363.324209:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764363.324211:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764363.324214:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764363.324217:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764363.324220:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764363.324221:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764363.324223:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764363.324226:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000040:3.0:1787764363.580219:0:304625:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35bb30 (cpu 3) has reached its deadline 4316750579, at 4316750592
00000100:00000010:3.0:1787764363.580231:0:304625:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4480.
00000100:00000040:2.0:1787764363.580250:0:304343:0:(connection.c:75:cpu_latency_work()) work item of ffff9751da35ba20 (cpu 2) has reached its deadline 4316750579, at 4316750592
00000100:00000010:2.0:1787764363.580260:0:304343:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9750e35c4d80.
00000100:00000001:2.0:1787764363.964180:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:2.0:1787764363.964187:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:2.0:1787764363.964189:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764363.964194:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00000100:00000001:2.0:1787764363.964198:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:2.0:1787764363.964201:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764363.964203:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764363.964205:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764363.964208:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764363.964247:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764363.964254:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764363.964260:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764363.964262:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764363.964265:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764363.964269:0:304625:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered
00010000:00000001:3.0:1787764363.964271:0:304625:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered
00010000:00000001:3.0:1787764363.964274:0:304625:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764363.964277:0:304625:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered
00010000:00000001:3.0:1787764363.964280:0:304625:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764363.964282:0:304625:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0)
00010000:00000001:3.0:1787764363.964284:0:304625:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764363.964350:0:282441:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764363.964355:0:282441:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764363.964358:0:282441:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764364.348226:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764364.348237:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764364.348240:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764364.348247:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764364.348252:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764364.348257:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764364.348260:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764364.348264:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764364.348267:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764364.988256:0:282442:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00010000:00000001:3.0:1787764364.988260:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764364.988266:0:282442:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00010000:00000001:3.0:1787764364.988266:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764364.988270:0:282442:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:2.0:1787764364.988275:0:282442:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751e05f6a80 going to sleep for 0 seconds
00010000:00000001:3.0:1787764364.988275:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00010000:00000001:3.0:1787764364.988278:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764364.988281:0:282442:0:(client.c:1990:ptlrpc_check_set()) Process entered
00010000:00000001:3.0:1787764364.988285:0:304625:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered
00000100:00000001:2.0:1787764364.988286:0:282442:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00010000:00000001:3.0:1787764364.988287:0:304625:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0)
00000100:00000001:2.0:1787764364.988291:0:282442:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:2.0:1787764364.988296:0:282442:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:2.0:1787764364.988299:0:282442:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
00000020:00000001:1.0:1787764364.988359:0:302605:0:(genops.c:1895:obd_stale_export_get()) Process entered
00000020:00000001:1.0:1787764364.988363:0:302605:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0)
00010000:00010000:1.0:1787764364.988367:0:302605:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue)
00000100:00000001:3.0:1787764365.372244:0:304093:0:(client.c:2567:ptlrpc_expired_set()) Process entered
00000100:00000001:3.0:1787764365.372251:0:304093:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered
00000100:00000001:3.0:1787764365.372254:0:304093:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0)
00000100:00100000:3.0:1787764365.372259:0:304093:0:(client.c:2715:ptlrpc_set_wait()) set ffff9751d0c54e40 going to sleep for 0 seconds
00000100:00000001:3.0:1787764365.372263:0:304093:0:(client.c:1990:ptlrpc_check_set()) Process entered
00000100:00000001:3.0:1787764365.372266:0:304093:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764365.372269:0:304093:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered
00000100:00000001:3.0:1787764365.372271:0:304093:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1)
00000100:00000001:3.0:1787764365.372274:0:304093:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0)
Debug log: 136382 lines, 136382 kept, 0 dropped, 0 bad.